diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 1618e793d..40c701217 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -18,6 +18,7 @@ from mbedtls_framework import macro_collector #pylint: disable=unused-import from mbedtls_framework import psa_information from mbedtls_framework import psa_storage +from mbedtls_framework import psa_test_case from mbedtls_framework import test_case from mbedtls_framework import test_data_generation @@ -32,17 +33,19 @@ def test_case_for_key_type_not_supported( """Return one test case exercising a key creation method for an unsupported key type or size. """ - psa_information.hack_dependencies_not_implemented(dependencies) - tc = test_case.TestCase() + tc = psa_test_case.TestCase() short_key_type = crypto_knowledge.short_expression(key_type) adverb = 'not' if dependencies else 'never' if param_descr: adverb = param_descr + ' ' + adverb tc.set_description('PSA {} {} {}-bit {} supported' .format(verb, short_key_type, bits, adverb)) - tc.set_dependencies(dependencies) tc.set_function(verb + '_not_supported') + tc.set_key_bits(bits) + tc.set_key_pair_usage(verb.upper()) tc.set_arguments([key_type] + list(args)) + tc.set_dependencies(dependencies) + tc.skip_if_any_not_implemented(dependencies) return tc class KeyTypeNotSupported: @@ -141,21 +144,19 @@ def test_cases_for_not_supported(self) -> Iterator[test_case.TestCase]: def test_case_for_key_generation( key_type: str, bits: int, - dependencies: List[str], *args: str, result: str = '' ) -> test_case.TestCase: """Return one test case exercising a key generation. """ - psa_information.hack_dependencies_not_implemented(dependencies) - tc = test_case.TestCase() + tc = psa_test_case.TestCase() short_key_type = crypto_knowledge.short_expression(key_type) tc.set_description('PSA {} {}-bit' .format(short_key_type, bits)) - tc.set_dependencies(dependencies) tc.set_function('generate_key') + tc.set_key_bits(bits) + tc.set_key_pair_usage('GENERATE') tc.set_arguments([key_type] + list(args) + [result]) - return tc class KeyGenerate: @@ -178,33 +179,18 @@ def test_cases_for_key_type_key_generation( All key types can be generated except for public keys. For public key PSA_ERROR_INVALID_ARGUMENT status is expected. """ - result = 'PSA_SUCCESS' - - import_dependencies = [psa_information.psa_want_symbol(kt.name)] - if kt.params is not None: - import_dependencies += [psa_information.psa_want_symbol(sym) - for i, sym in enumerate(kt.params)] - if kt.name.endswith('_PUBLIC_KEY'): - # The library checks whether the key type is a public key generically, - # before it reaches a point where it needs support for the specific key - # type, so it returns INVALID_ARGUMENT for unsupported public key types. - generate_dependencies = [] - result = 'PSA_ERROR_INVALID_ARGUMENT' - else: - generate_dependencies = \ - psa_information.fix_key_pair_dependencies(import_dependencies, 'GENERATE') for bits in kt.sizes_to_test(): - if kt.name == 'PSA_KEY_TYPE_RSA_KEY_PAIR': - size_dependency = "PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS <= " + str(bits) - test_dependencies = generate_dependencies + [size_dependency] - else: - test_dependencies = generate_dependencies - yield test_case_for_key_generation( + tc = test_case_for_key_generation( kt.expression, bits, - psa_information.finish_family_dependencies(test_dependencies, bits), str(bits), - result + 'PSA_ERROR_INVALID_ARGUMENT' if kt.is_public() else 'PSA_SUCCESS' ) + if kt.is_public(): + # The library checks whether the key type is a public key generically, + # before it reaches a point where it needs support for the specific key + # type, so it returns INVALID_ARGUMENT for unsupported public key types. + tc.set_dependencies([]) + yield tc def test_cases_for_key_generation(self) -> Iterator[test_case.TestCase]: """Generate test cases that exercise the generation of keys.""" @@ -252,7 +238,7 @@ def make_test_case( ) -> test_case.TestCase: """Construct a failure test case for a one-key or keyless operation.""" #pylint: disable=too-many-arguments,too-many-locals - tc = test_case.TestCase() + tc = psa_test_case.TestCase() pretty_alg = alg.short_expression() if reason == self.Reason.NOT_SUPPORTED: short_deps = [re.sub(r'PSA_WANT_ALG_', r'', dep) @@ -276,11 +262,12 @@ def make_test_case( for i, dep in enumerate(dependencies): if dep in not_deps: dependencies[i] = '!' + dep - tc.set_dependencies(dependencies) tc.set_function(category.name.lower() + '_fail') arguments = [] # type: List[str] if kt: - key_material = kt.key_material(kt.sizes_to_test()[0]) + bits = kt.sizes_to_test()[0] + tc.set_key_bits(bits) + key_material = kt.key_material(bits) arguments += [key_type, test_case.hex_string(key_material)] arguments.append(alg.expression) if category.is_asymmetric(): @@ -289,6 +276,7 @@ def make_test_case( 'INVALID_ARGUMENT') arguments.append('PSA_ERROR_' + error) tc.set_arguments(arguments) + tc.set_dependencies(dependencies) return tc def no_key_test_cases( @@ -488,17 +476,12 @@ def make_test_case(self, key: StorageTestData) -> test_case.TestCase: correctly. """ verb = 'save' if self.forward else 'read' - tc = test_case.TestCase() + tc = psa_test_case.TestCase() tc.set_description(verb + ' ' + key.description) - dependencies = psa_information.automatic_dependencies( - key.lifetime.string, key.type.string, - key.alg.string, key.alg2.string, - ) - dependencies = psa_information.finish_family_dependencies(dependencies, key.bits) - dependencies += psa_information.generate_deps_from_description(key.description) - dependencies = psa_information.fix_key_pair_dependencies(dependencies, 'BASIC') - tc.set_dependencies(dependencies) + tc.add_dependencies(psa_information.generate_deps_from_description(key.description)) tc.set_function('key_storage_' + verb) + tc.set_key_bits(key.bits) + tc.set_key_pair_usage('BASIC') if self.forward: extra_arguments = [] else: diff --git a/scripts/mbedtls_framework/crypto_data_tests.py b/scripts/mbedtls_framework/crypto_data_tests.py index a36de692e..1d46e3f2c 100644 --- a/scripts/mbedtls_framework/crypto_data_tests.py +++ b/scripts/mbedtls_framework/crypto_data_tests.py @@ -12,20 +12,10 @@ from . import crypto_knowledge from . import psa_information +from . import psa_test_case from . import test_case -def psa_low_level_dependencies(*expressions: str) -> List[str]: - """Infer dependencies of a PSA low-level test case by looking for PSA_xxx symbols. - - This function generates MBEDTLS_PSA_BUILTIN_xxx symbols. - """ - high_level = psa_information.automatic_dependencies(*expressions) - for dep in high_level: - assert dep.startswith('PSA_WANT_') - return ['MBEDTLS_PSA_BUILTIN_' + dep[9:] for dep in high_level] - - class HashPSALowLevel: """Generate test cases for the PSA low-level hash interface.""" @@ -69,12 +59,11 @@ def one_test_case(alg: crypto_knowledge.Algorithm, function: str, note: str, arguments: List[str]) -> test_case.TestCase: """Construct one test case involving a hash.""" - tc = test_case.TestCase() + tc = psa_test_case.TestCase(dependency_prefix='MBEDTLS_PSA_BUILTIN_') tc.set_description('{}{} {}' .format(function, ' ' + note if note else '', alg.short_expression())) - tc.set_dependencies(psa_low_level_dependencies(alg.expression)) tc.set_function(function) tc.set_arguments([alg.expression] + ['"{}"'.format(arg) for arg in arguments]) diff --git a/scripts/mbedtls_framework/psa_information.py b/scripts/mbedtls_framework/psa_information.py index 0d4ea9edd..1ff02da61 100644 --- a/scripts/mbedtls_framework/psa_information.py +++ b/scripts/mbedtls_framework/psa_information.py @@ -8,7 +8,7 @@ import os import re from collections import OrderedDict -from typing import FrozenSet, List, Optional +from typing import List, Optional from . import macro_collector @@ -23,8 +23,13 @@ def __init__(self) -> None: def remove_unwanted_macros( constructors: macro_collector.PSAMacroEnumerator ) -> None: - # Mbed TLS does not support finite-field DSA. + """Remove constructors that should be exckuded from systematic testing.""" + # Mbed TLS does not support finite-field DSA, but 3.6 defines DSA + # identifiers for historical reasons. # Don't attempt to generate any related test case. + # The corresponding test cases would be commented out anyway, + # but for DSA, we don't have enough support in the test scripts + # to generate these test cases. constructors.key_types.discard('PSA_KEY_TYPE_DSA_KEY_PAIR') constructors.key_types.discard('PSA_KEY_TYPE_DSA_PUBLIC_KEY') @@ -52,10 +57,16 @@ def read_psa_interface(self) -> macro_collector.PSAMacroEnumerator: return constructors -def psa_want_symbol(name: str) -> str: - """Return the PSA_WANT_xxx symbol associated with a PSA crypto feature.""" +def psa_want_symbol(name: str, prefix: Optional[str] = None) -> str: + """Return the PSA_WANT_xxx symbol associated with a PSA crypto feature. + + You can use an altenative `prefix`, e.g. 'MBEDTLS_PSA_BUILTIN_' + when specifically testing builtin implementations. + """ + if prefix is None: + prefix = 'PSA_WANT_' if name.startswith('PSA_'): - return name[:4] + 'WANT_' + name[4:] + return prefix + name[4:] else: raise ValueError('Unable to determine the PSA_WANT_ symbol for ' + name) @@ -83,18 +94,23 @@ def finish_family_dependencies(dependencies: List[str], bits: int) -> List[str]: 'PSA_ALG_KEY_AGREEMENT', # chaining 'PSA_ALG_TRUNCATED_MAC', # modifier ]) -def automatic_dependencies(*expressions: str) -> List[str]: +def automatic_dependencies(*expressions: str, + prefix: Optional[str] = None) -> List[str]: """Infer dependencies of a test case by looking for PSA_xxx symbols. The arguments are strings which should be C expressions. Do not use string literals or comments as this function is not smart enough to skip them. + + `prefix`: prefix to use in dependencies. Defaults to ``'PSA_WANT_'``. + Use ``'MBEDTLS_PSA_BUILTIN_'`` when specifically testing + builtin implementations. """ used = set() for expr in expressions: used.update(re.findall(r'PSA_(?:ALG|ECC_FAMILY|DH_FAMILY|KEY_TYPE)_\w+', expr)) used.difference_update(SYMBOLS_WITHOUT_DEPENDENCY) - return sorted(psa_want_symbol(name) for name in used) + return sorted(psa_want_symbol(name, prefix=prefix) for name in used) # Define set of regular expressions and dependencies to optionally append # extra dependencies for test case based on key description. @@ -123,38 +139,6 @@ def generate_deps_from_description( return dep_list -# A temporary hack: at the time of writing, not all dependency symbols -# are implemented yet. Skip test cases for which the dependency symbols are -# not available. Once all dependency symbols are available, this hack must -# be removed so that a bug in the dependency symbols properly leads to a test -# failure. -def read_implemented_dependencies(filename: str) -> FrozenSet[str]: - return frozenset(symbol - for line in open(filename) - for symbol in re.findall(r'\bPSA_WANT_\w+\b', line)) -_implemented_dependencies = None #type: Optional[FrozenSet[str]] #pylint: disable=invalid-name -def hack_dependencies_not_implemented(dependencies: List[str]) -> None: - """ - Hack dependencies to skip test cases for which at least one dependency - symbol is not available yet. - """ - global _implemented_dependencies #pylint: disable=global-statement,invalid-name - if _implemented_dependencies is None: - # Temporary, while Mbed TLS does not just rely on the TF-PSA-Crypto - # build system to build its crypto library. When it does, the first - # case can just be removed. - if os.path.isdir('tf-psa-crypto'): - _implemented_dependencies = \ - read_implemented_dependencies('tf-psa-crypto/include/psa/crypto_config.h') - else: - _implemented_dependencies = \ - read_implemented_dependencies('include/psa/crypto_config.h') - - if not all((dep.lstrip('!') in _implemented_dependencies or - not dep.lstrip('!').startswith('PSA_WANT')) - for dep in dependencies): - dependencies.append('DEPENDENCY_NOT_IMPLEMENTED_YET') - def tweak_key_pair_dependency(dep: str, usage: str): """ This helper function add the proper suffix to PSA_WANT_KEY_TYPE_xxx_KEY_PAIR diff --git a/scripts/mbedtls_framework/psa_test_case.py b/scripts/mbedtls_framework/psa_test_case.py new file mode 100644 index 000000000..eea969f7a --- /dev/null +++ b/scripts/mbedtls_framework/psa_test_case.py @@ -0,0 +1,145 @@ +"""Generate test cases for PSA API calls, with automatic dependencies. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# + +import os +import re +from typing import FrozenSet, List, Optional, Set + +from . import psa_information +from . import test_case + + +# A temporary hack: at the time of writing, not all dependency symbols +# are implemented yet. Skip test cases for which the dependency symbols are +# not available. Once all dependency symbols are available, this hack must +# be removed so that a bug in the dependency symbols properly leads to a test +# failure. +def read_implemented_dependencies(acc: Set[str], filename: str) -> None: + with open(filename) as input_stream: + for line in input_stream: + for symbol in re.findall(r'\bPSA_WANT_\w+\b', line): + acc.add(symbol) + +_implemented_dependencies = None #type: Optional[FrozenSet[str]] #pylint: disable=invalid-name + +def find_dependencies_not_implemented(dependencies: List[str]) -> List[str]: + """List the dependencies that are not implemented.""" + global _implemented_dependencies #pylint: disable=global-statement,invalid-name + if _implemented_dependencies is None: + # Temporary, while Mbed TLS does not just rely on the TF-PSA-Crypto + # build system to build its crypto library. When it does, the first + # case can just be removed. + if os.path.isdir('tf-psa-crypto'): + include_dir = 'tf-psa-crypto/include' + else: + include_dir = 'include' + acc = set() #type: Set[str] + for filename in [ + os.path.join(include_dir, 'psa/crypto_config.h'), + os.path.join(include_dir, 'psa/crypto_adjust_config_synonyms.h'), + ]: + read_implemented_dependencies(acc, filename) + _implemented_dependencies = frozenset(acc) + return [dep + for dep in dependencies + if (dep.lstrip('!') not in _implemented_dependencies and + dep.lstrip('!').startswith('PSA_WANT'))] + + +class TestCase(test_case.TestCase): + """A PSA test case with automatically inferred dependencies. + + For mechanisms like ECC curves where the support status includes + the key bit-size, this class assumes that only one bit-size is + involved in a given test case. + """ + + def __init__(self, dependency_prefix: Optional[str] = None) -> None: + """Construct a test case for a PSA Crypto API call. + + `dependency_prefix`: prefix to use in dependencies. Defaults to + ``'PSA_WANT_'``. Use ``'MBEDTLS_PSA_BUILTIN_'`` + when specifically testing builtin implementations. + """ + super().__init__() + del self.dependencies + self.manual_dependencies = [] #type: List[str] + self.automatic_dependencies = set() #type: Set[str] + self.dependency_prefix = dependency_prefix #type: Optional[str] + self.key_bits = None #type: Optional[int] + self.key_pair_usage = None #type: Optional[str] + + def set_key_bits(self, key_bits: Optional[int]) -> None: + """Use the given key size for automatic dependency generation. + + Call this function before set_arguments() if relevant. + + This is only relevant for ECC and DH keys. For other key types, + this information is ignored. + """ + self.key_bits = key_bits + + def set_key_pair_usage(self, key_pair_usage: Optional[str]) -> None: + """Use the given suffix for key pair dependencies. + + Call this function before set_arguments() if relevant. + + This is only relevant for key pair types. For other key types, + this information is ignored. + """ + self.key_pair_usage = key_pair_usage + + def infer_dependencies(self, arguments: List[str]) -> List[str]: + """Infer dependencies based on the test case arguments.""" + dependencies = psa_information.automatic_dependencies(*arguments, + prefix=self.dependency_prefix) + if self.key_bits is not None: + dependencies = psa_information.finish_family_dependencies(dependencies, + self.key_bits) + if self.key_pair_usage is not None: + dependencies = psa_information.fix_key_pair_dependencies(dependencies, + self.key_pair_usage) + if 'PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE' in dependencies and \ + self.key_bits is not None: + size_dependency = ('PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS <= ' + + str(self.key_bits)) + dependencies.append(size_dependency) + return dependencies + + def set_arguments(self, arguments: List[str]) -> None: + """Set test case arguments and automatically infer dependencies.""" + super().set_arguments(arguments) + dependencies = self.infer_dependencies(arguments) + self.skip_if_any_not_implemented(dependencies) + self.automatic_dependencies.update(dependencies) + + def set_dependencies(self, dependencies: List[str]) -> None: + """Override any previously added automatic or manual dependencies. + + Also override any previous instruction to skip the test case. + """ + self.manual_dependencies = dependencies + self.automatic_dependencies.clear() + self.skip_reasons = [] + + def add_dependencies(self, dependencies: List[str]) -> None: + """Add manual dependencies.""" + self.manual_dependencies += dependencies + + def get_dependencies(self) -> List[str]: + # Make the output independent of the order in which the dependencies + # are calculated by the script. Also avoid duplicates. This makes + # the output robust with respect to refactoring of the scripts. + dependencies = set(self.manual_dependencies) + dependencies.update(self.automatic_dependencies) + return sorted(dependencies) + + def skip_if_any_not_implemented(self, dependencies: List[str]) -> None: + """Skip the test case if any of the given dependencies is not implemented.""" + not_implemented = find_dependencies_not_implemented(dependencies) + for dep in not_implemented: + self.skip_because('not implemented: ' + dep) diff --git a/scripts/mbedtls_framework/test_case.py b/scripts/mbedtls_framework/test_case.py index 47a5e4f7d..58e75a1ca 100644 --- a/scripts/mbedtls_framework/test_case.py +++ b/scripts/mbedtls_framework/test_case.py @@ -57,6 +57,7 @@ def __init__(self, description: Optional[str] = None): self.dependencies = [] #type: List[str] self.function = None #type: Optional[str] self.arguments = [] #type: List[str] + self.skip_reasons = [] #type: List[str] def add_comment(self, *lines: str) -> None: self.comments += lines @@ -64,6 +65,9 @@ def add_comment(self, *lines: str) -> None: def set_description(self, description: str) -> None: self.description = description + def get_dependencies(self) -> List[str]: + return self.dependencies + def set_dependencies(self, dependencies: List[str]) -> None: self.dependencies = dependencies @@ -73,6 +77,23 @@ def set_function(self, function: str) -> None: def set_arguments(self, arguments: List[str]) -> None: self.arguments = arguments + def skip_because(self, reason: str) -> None: + """Skip this test case. + + It will be included in the output, but commented out. + + This is intended for test cases that are obtained from a + systematic enumeration, but that have dependencies that cannot + be fulfilled. Since we don't want to have test cases that are + never executed, we arrange not to have actual test cases. But + we do include comments to make it easier to understand the output + of test case generation. + + reason must be a non-empty string explaining to humans why this + test case is skipped. + """ + self.skip_reasons.append(reason) + def check_completeness(self) -> None: if self.description is None: raise MissingDescription @@ -93,10 +114,18 @@ def write(self, out: typing_util.Writable) -> None: out.write('\n') for line in self.comments: out.write('# ' + line + '\n') - out.write(self.description + '\n') - if self.dependencies: - out.write('depends_on:' + ':'.join(self.dependencies) + '\n') - out.write(self.function + ':' + ':'.join(self.arguments) + '\n') + prefix = '' + if self.skip_reasons: + prefix = '## ' + for reason in self.skip_reasons: + out.write('## # skipped because: ' + reason + '\n') + out.write(prefix + self.description + '\n') + dependencies = self.get_dependencies() + if dependencies: + out.write(prefix + 'depends_on:' + + ':'.join(dependencies) + '\n') + out.write(prefix + self.function + ':' + + ':'.join(self.arguments) + '\n') def write_data_file(filename: str, test_cases: Iterable[TestCase],