Report configuration settings in the outcome file

scripts/config.py

@@ -396,6 +396,7 @@ def __init__(self, filename=None):
                                 self.default_path)
         super().__init__()
         self.filename = filename
+        self.inclusion_guard = None
         self.current_section = 'header'
         with open(filename, 'r', encoding='utf-8') as file:
             self.templates = [self._parse_line(line) for line in file]
@@ -413,9 +414,11 @@ def set(self, name, value=None):
                            r'(?P<arguments>(?:\((?:\w|\s|,)*\))?)' +
                            r'(?P<separator>\s*)' +
                            r'(?P<value>.*)')
+    _ifndef_line_regexp = r'#ifndef (?P<inclusion_guard>\w+)'
     _section_line_regexp = (r'\s*/?\*+\s*[\\@]name\s+SECTION:\s*' +
                             r'(?P<section>.*)[ */]*')
     _config_line_regexp = re.compile(r'|'.join([_define_line_regexp,
+                                                _ifndef_line_regexp,
                                                 _section_line_regexp]))
     def _parse_line(self, line):
         """Parse a line in mbedtls_config.h and return the corresponding template."""
@@ -426,10 +429,16 @@ def _parse_line(self, line):
         elif m.group('section'):
             self.current_section = m.group('section')
             return line
+        elif m.group('inclusion_guard') and self.inclusion_guard is None:
+            self.inclusion_guard = m.group('inclusion_guard')
+            return line
         else:
             active = not m.group('commented_out')
             name = m.group('name')
             value = m.group('value')
+            if name == self.inclusion_guard and value == '':
+                # The file double-inclusion guard is not an option.
+                return line
             template = (name,
                         m.group('indentation'),
                         m.group('define') + name +

scripts/make_generated_files.bat

@@ -11,6 +11,7 @@ python scripts\generate_ssl_debug_helpers.py || exit /b 1
 perl scripts\generate_visualc_files.pl || exit /b 1
 python scripts\generate_psa_constants.py || exit /b 1
 python framework\scripts\generate_bignum_tests.py || exit /b 1
+python framework\scripts\generate_config_tests.py || exit /b 1
 python framework\scripts\generate_ecp_tests.py || exit /b 1
 python framework\scripts\generate_psa_tests.py || exit /b 1
 python framework\scripts\generate_test_keys.py --output tests\src\test_keys.h || exit /b 1

tests/.gitignore

@@ -3,22 +3,24 @@
 
 *.log
 /test_suite*
-data_files/mpi_write
-data_files/hmac_drbg_seed
-data_files/ctr_drbg_seed
-data_files/entropy_seed
+/data_files/mpi_write
+/data_files/hmac_drbg_seed
+/data_files/ctr_drbg_seed
+/data_files/entropy_seed
 
-include/alt-extra/psa/crypto_platform_alt.h
-include/alt-extra/psa/crypto_struct_alt.h
-include/test/instrument_record_status.h
+/include/alt-extra/psa/crypto_platform_alt.h
+/include/alt-extra/psa/crypto_struct_alt.h
+/include/test/instrument_record_status.h
 
-src/libmbed*
+/src/libmbed*
 
-libtestdriver1/*
+/libtestdriver1/*
 
 ###START_GENERATED_FILES###
 # Generated source files
 /suites/*.generated.data
+/suites/test_suite_config.mbedtls_boolean.data
+/suites/test_suite_config.psa_boolean.data
 /suites/test_suite_psa_crypto_storage_format.v[0-9]*.data
 /suites/test_suite_psa_crypto_storage_format.current.data
 /src/test_keys.h

tests/CMakeLists.txt

@@ -30,6 +30,18 @@ execute_process(
 string(REGEX REPLACE "[^;]*/" ""
        base_bignum_generated_data_files "${base_bignum_generated_data_files}")
 
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+        --list-for-cmake
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_config_generated_data_files)
+string(REGEX REPLACE "[^;]*/" ""
+       base_config_generated_data_files "${base_config_generated_data_files}")
+
 execute_process(
     COMMAND
         ${MBEDTLS_PYTHON_EXECUTABLE}
@@ -61,11 +73,15 @@ set(base_generated_data_files
 string(REGEX REPLACE "([^;]+)" "suites/\\1"
        all_generated_data_files "${base_generated_data_files}")
 set(bignum_generated_data_files "")
+set(config_generated_data_files "")
 set(ecp_generated_data_files "")
 set(psa_generated_data_files "")
 foreach(file ${base_bignum_generated_data_files})
     list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
 endforeach()
+foreach(file ${base_config_generated_data_files})
+    list(APPEND config_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
+endforeach()
 foreach(file ${base_ecp_generated_data_files})
     list(APPEND ecp_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
 endforeach()
@@ -92,6 +108,21 @@ if(GEN_FILES)
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_case.py
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_data_generation.py
     )
+    add_custom_command(
+        OUTPUT
+            ${config_generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+            # Do not declare the configuration files as dependencies: they
+            # change too often in ways that don't affect the result
+            # ((un)commenting some options).
+    )
     add_custom_command(
         OUTPUT
             ${ecp_generated_data_files}
@@ -142,6 +173,7 @@ endif()
 # With this line, only 4 sub-makefiles include the above command, that reduces
 # the risk of a race.
 add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files})
+add_custom_target(test_suite_config_generated_data DEPENDS ${config_generated_data_files})
 add_custom_target(test_suite_ecp_generated_data DEPENDS ${ecp_generated_data_files})
 add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files})
 # If SKIP_TEST_SUITES is not defined with -D, get it from the environment.
@@ -199,6 +231,10 @@ function(add_test_suite suite_name)
         set(data_file
             ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
         set(dependency test_suite_bignum_generated_data)
+    elseif(";${config_generated_data_names};" MATCHES ";${data_name};")
+        set(data_file
+            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_bignum_generated_data)
     elseif(";${ecp_generated_data_names};" MATCHES ";${data_name};")
         set(data_file
             ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
@@ -210,7 +246,11 @@ function(add_test_suite suite_name)
     else()
         set(data_file
             ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data)
-        set(dependency test_suite_bignum_generated_data test_suite_ecp_generated_data test_suite_psa_generated_data)
+        set(dependency
+            test_suite_bignum_generated_data
+            test_suite_config_generated_data
+            test_suite_ecp_generated_data
+            test_suite_psa_generated_data)
     endif()
 
     add_custom_command(

tests/Makefile

@@ -17,38 +17,54 @@ ifdef RECORD_PSA_STATUS_COVERAGE_LOG
 LOCAL_CFLAGS += -Werror -DRECORD_PSA_STATUS_COVERAGE_LOG
 endif
 
-.PHONY: generated_files
 GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \
 	$(PYTHON) ../framework/scripts/generate_bignum_tests.py --list || \
 	echo FAILED \
 ))
 ifeq ($(GENERATED_BIGNUM_DATA_FILES),FAILED)
 $(error "$(PYTHON) ../framework/scripts/generate_bignum_tests.py --list" failed)
 endif
+GENERATED_DATA_FILES += $(GENERATED_BIGNUM_DATA_FILES)
+
+GENERATED_CONFIG_DATA_FILES := $(patsubst tests/%,%,$(shell \
+	$(PYTHON) ../framework/scripts/generate_config_tests.py --list || \
+	echo FAILED \
+))
+ifeq ($(GENERATED_CONFIG_DATA_FILES),FAILED)
+$(error "$(PYTHON) ../framework/scripts/generate_config_tests.py --list" failed)
+endif
+GENERATED_DATA_FILES += $(GENERATED_CONFIG_DATA_FILES)
+
 GENERATED_ECP_DATA_FILES := $(patsubst tests/%,%,$(shell \
 	$(PYTHON) ../framework/scripts/generate_ecp_tests.py --list || \
 	echo FAILED \
 ))
 ifeq ($(GENERATED_ECP_DATA_FILES),FAILED)
 $(error "$(PYTHON) ../framework/scripts/generate_ecp_tests.py --list" failed)
 endif
+GENERATED_DATA_FILES += $(GENERATED_ECP_DATA_FILES)
+
 GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \
 	$(PYTHON) ../framework/scripts/generate_psa_tests.py --list || \
 	echo FAILED \
 ))
 ifeq ($(GENERATED_PSA_DATA_FILES),FAILED)
 $(error "$(PYTHON) ../framework/scripts/generate_psa_tests.py --list" failed)
 endif
-GENERATED_FILES := $(GENERATED_PSA_DATA_FILES) $(GENERATED_ECP_DATA_FILES) $(GENERATED_BIGNUM_DATA_FILES)
-generated_files: $(GENERATED_FILES) src/test_keys.h src/test_certs.h
+GENERATED_DATA_FILES += $(GENERATED_PSA_DATA_FILES)
+
+GENERATED_FILES = $(GENERATED_DATA_FILES)
+GENERATED_FILES += src/test_keys.h src/test_certs.h
+
+.PHONY: generated_files
+generated_files: $(GENERATED_FILES)
 
 # generate_bignum_tests.py and generate_psa_tests.py spend more time analyzing
 # inputs than generating outputs. Its inputs are the same no matter which files
 # are being generated.
 # It's rare not to want all the outputs. So always generate all of its outputs.
 # Use an intermediate phony dependency so that parallel builds don't run
 # a separate instance of the recipe for each output file.
-.SECONDARY: generated_bignum_test_data generated_ecp_test_data generated_psa_test_data
 $(GENERATED_BIGNUM_DATA_FILES): $(gen_file_dep) generated_bignum_test_data
 generated_bignum_test_data: ../framework/scripts/generate_bignum_tests.py
 generated_bignum_test_data: ../framework/scripts/mbedtls_framework/bignum_common.py
@@ -60,6 +76,23 @@ generated_bignum_test_data: ../framework/scripts/mbedtls_framework/test_data_gen
 generated_bignum_test_data:
 	echo "  Gen   $(GENERATED_BIGNUM_DATA_FILES)"
 	$(PYTHON) ../framework/scripts/generate_bignum_tests.py
+.SECONDARY: generated_bignum_test_data
+
+# We deliberately omit the configuration files (mbedtls_config.h,
+# crypto_config.h) from the depenency list because during development
+# and on the CI, we often edit those in a way that doesn't change the
+# output, to comment out certain options, or even to remove certain
+# lines which do affect the output negatively (it will miss the
+# corresponding test cases).
+$(GENERATED_CONFIG_DATA_FILES): $(gen_file_dep) generated_config_test_data
+generated_config_test_data: ../framework/scripts/generate_config_tests.py
+generated_config_test_data: ../scripts/config.py
+generated_config_test_data: ../framework/scripts/mbedtls_framework/test_case.py
+generated_config_test_data: ../framework/scripts/mbedtls_framework/test_data_generation.py
+generated_config_test_data:
+	echo "  Gen   $(GENERATED_CONFIG_DATA_FILES)"
+	$(PYTHON) ../framework/scripts/generate_config_tests.py
+.SECONDARY: generated_config_test_data
 
 $(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data
 generated_ecp_test_data: ../framework/scripts/generate_ecp_tests.py
@@ -70,6 +103,7 @@ generated_ecp_test_data: ../framework/scripts/mbedtls_framework/test_data_genera
 generated_ecp_test_data:
 	echo "  Gen   $(GENERATED_ECP_DATA_FILES)"
 	$(PYTHON) ../framework/scripts/generate_ecp_tests.py
+.SECONDARY: generated_ecp_test_data
 
 $(GENERATED_PSA_DATA_FILES): $(gen_file_dep) generated_psa_test_data
 generated_psa_test_data: ../framework/scripts/generate_psa_tests.py
@@ -92,14 +126,15 @@ generated_psa_test_data: suites/test_suite_psa_crypto_metadata.data
 generated_psa_test_data:
 	echo "  Gen   $(GENERATED_PSA_DATA_FILES) ..."
 	$(PYTHON) ../framework/scripts/generate_psa_tests.py
+.SECONDARY: generated_psa_test_data
 
 # A test application is built for each suites/test_suite_*.data file.
 # Application name is same as .data file's base name and can be
 # constructed by stripping path 'suites/' and extension .data.
 DATA_FILES := $(wildcard suites/test_suite_*.data)
 # Make sure that generated data files are included even if they don't
 # exist yet when the makefile is parsed.
-DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_FILES))
+DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_DATA_FILES))
 APPS = $(basename $(subst suites/,,$(DATA_FILES)))
 
 # Construct executable name by adding OS specific suffix $(EXEXT).