Switch generate_psa_test.py to automatic dependencies for positive test cases

scripts/generate_psa_tests.py

@@ -18,6 +18,7 @@
 from mbedtls_framework import macro_collector #pylint: disable=unused-import
 from mbedtls_framework import psa_information
 from mbedtls_framework import psa_storage
+from mbedtls_framework import psa_test_case
 from mbedtls_framework import test_case
 from mbedtls_framework import test_data_generation
 
@@ -32,17 +33,19 @@ def test_case_for_key_type_not_supported(
     """Return one test case exercising a key creation method
     for an unsupported key type or size.
     """
-    psa_information.hack_dependencies_not_implemented(dependencies)
-    tc = test_case.TestCase()
+    tc = psa_test_case.TestCase()
     short_key_type = crypto_knowledge.short_expression(key_type)
     adverb = 'not' if dependencies else 'never'
     if param_descr:
         adverb = param_descr + ' ' + adverb
     tc.set_description('PSA {} {} {}-bit {} supported'
                        .format(verb, short_key_type, bits, adverb))
-    tc.set_dependencies(dependencies)
     tc.set_function(verb + '_not_supported')
+    tc.set_key_bits(bits)
+    tc.set_key_pair_usage(verb.upper())
     tc.set_arguments([key_type] + list(args))
+    tc.set_dependencies(dependencies)
+    tc.skip_if_any_not_implemented(dependencies)
     return tc
 
 class KeyTypeNotSupported:
@@ -141,21 +144,19 @@ def test_cases_for_not_supported(self) -> Iterator[test_case.TestCase]:
 
 def test_case_for_key_generation(
         key_type: str, bits: int,
-        dependencies: List[str],
         *args: str,
         result: str = ''
 ) -> test_case.TestCase:
     """Return one test case exercising a key generation.
     """
-    psa_information.hack_dependencies_not_implemented(dependencies)
-    tc = test_case.TestCase()
+    tc = psa_test_case.TestCase()
     short_key_type = crypto_knowledge.short_expression(key_type)
     tc.set_description('PSA {} {}-bit'
                        .format(short_key_type, bits))
-    tc.set_dependencies(dependencies)
     tc.set_function('generate_key')
+    tc.set_key_bits(bits)
+    tc.set_key_pair_usage('GENERATE')
     tc.set_arguments([key_type] + list(args) + [result])
-
     return tc
 
 class KeyGenerate:
@@ -178,33 +179,18 @@ def test_cases_for_key_type_key_generation(
         All key types can be generated except for public keys. For public key
         PSA_ERROR_INVALID_ARGUMENT status is expected.
         """
-        result = 'PSA_SUCCESS'
-
-        import_dependencies = [psa_information.psa_want_symbol(kt.name)]
-        if kt.params is not None:
-            import_dependencies += [psa_information.psa_want_symbol(sym)
-                                    for i, sym in enumerate(kt.params)]
-        if kt.name.endswith('_PUBLIC_KEY'):
-            # The library checks whether the key type is a public key generically,
-            # before it reaches a point where it needs support for the specific key
-            # type, so it returns INVALID_ARGUMENT for unsupported public key types.
-            generate_dependencies = []
-            result = 'PSA_ERROR_INVALID_ARGUMENT'
-        else:
-            generate_dependencies = \
-                psa_information.fix_key_pair_dependencies(import_dependencies, 'GENERATE')
         for bits in kt.sizes_to_test():
-            if kt.name == 'PSA_KEY_TYPE_RSA_KEY_PAIR':
-                size_dependency = "PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS <= " +  str(bits)
-                test_dependencies = generate_dependencies + [size_dependency]
-            else:
-                test_dependencies = generate_dependencies
-            yield test_case_for_key_generation(
+            tc = test_case_for_key_generation(
                 kt.expression, bits,
-                psa_information.finish_family_dependencies(test_dependencies, bits),
                 str(bits),
-                result
+                'PSA_ERROR_INVALID_ARGUMENT' if kt.is_public() else 'PSA_SUCCESS'
             )
+            if kt.is_public():
+                # The library checks whether the key type is a public key generically,
+                # before it reaches a point where it needs support for the specific key
+                # type, so it returns INVALID_ARGUMENT for unsupported public key types.
+                tc.set_dependencies([])
+            yield tc
 
     def test_cases_for_key_generation(self) -> Iterator[test_case.TestCase]:
         """Generate test cases that exercise the generation of keys."""
@@ -252,7 +238,7 @@ def make_test_case(
     ) -> test_case.TestCase:
         """Construct a failure test case for a one-key or keyless operation."""
         #pylint: disable=too-many-arguments,too-many-locals
-        tc = test_case.TestCase()
+        tc = psa_test_case.TestCase()
         pretty_alg = alg.short_expression()
         if reason == self.Reason.NOT_SUPPORTED:
             short_deps = [re.sub(r'PSA_WANT_ALG_', r'', dep)
@@ -276,11 +262,12 @@ def make_test_case(
         for i, dep in enumerate(dependencies):
             if dep in not_deps:
                 dependencies[i] = '!' + dep
-        tc.set_dependencies(dependencies)
         tc.set_function(category.name.lower() + '_fail')
         arguments = [] # type: List[str]
         if kt:
-            key_material = kt.key_material(kt.sizes_to_test()[0])
+            bits = kt.sizes_to_test()[0]
+            tc.set_key_bits(bits)
+            key_material = kt.key_material(bits)
             arguments += [key_type, test_case.hex_string(key_material)]
         arguments.append(alg.expression)
         if category.is_asymmetric():
@@ -289,6 +276,7 @@ def make_test_case(
                  'INVALID_ARGUMENT')
         arguments.append('PSA_ERROR_' + error)
         tc.set_arguments(arguments)
+        tc.set_dependencies(dependencies)
         return tc
 
     def no_key_test_cases(
@@ -488,17 +476,12 @@ def make_test_case(self, key: StorageTestData) -> test_case.TestCase:
         correctly.
         """
         verb = 'save' if self.forward else 'read'
-        tc = test_case.TestCase()
+        tc = psa_test_case.TestCase()
         tc.set_description(verb + ' ' + key.description)
-        dependencies = psa_information.automatic_dependencies(
-            key.lifetime.string, key.type.string,
-            key.alg.string, key.alg2.string,
-        )
-        dependencies = psa_information.finish_family_dependencies(dependencies, key.bits)
-        dependencies += psa_information.generate_deps_from_description(key.description)
-        dependencies = psa_information.fix_key_pair_dependencies(dependencies, 'BASIC')
-        tc.set_dependencies(dependencies)
+        tc.add_dependencies(psa_information.generate_deps_from_description(key.description))
         tc.set_function('key_storage_' + verb)
+        tc.set_key_bits(key.bits)
+        tc.set_key_pair_usage('BASIC')
         if self.forward:
             extra_arguments = []
         else:

scripts/mbedtls_framework/crypto_data_tests.py

@@ -12,20 +12,10 @@
 
 from . import crypto_knowledge
 from . import psa_information
+from . import psa_test_case
 from . import test_case
 
 
-def psa_low_level_dependencies(*expressions: str) -> List[str]:
-    """Infer dependencies of a PSA low-level test case by looking for PSA_xxx symbols.
-
-    This function generates MBEDTLS_PSA_BUILTIN_xxx symbols.
-    """
-    high_level = psa_information.automatic_dependencies(*expressions)
-    for dep in high_level:
-        assert dep.startswith('PSA_WANT_')
-    return ['MBEDTLS_PSA_BUILTIN_' + dep[9:] for dep in high_level]
-
-
 class HashPSALowLevel:
     """Generate test cases for the PSA low-level hash interface."""
 
@@ -69,12 +59,11 @@ def one_test_case(alg: crypto_knowledge.Algorithm,
                       function: str, note: str,
                       arguments: List[str]) -> test_case.TestCase:
         """Construct one test case involving a hash."""
-        tc = test_case.TestCase()
+        tc = psa_test_case.TestCase(dependency_prefix='MBEDTLS_PSA_BUILTIN_')
         tc.set_description('{}{} {}'
                            .format(function,
                                    ' ' + note if note else '',
                                    alg.short_expression()))
-        tc.set_dependencies(psa_low_level_dependencies(alg.expression))
         tc.set_function(function)
         tc.set_arguments([alg.expression] +
                          ['"{}"'.format(arg) for arg in arguments])

scripts/mbedtls_framework/psa_information.py

@@ -8,7 +8,7 @@
 import os
 import re
 from collections import OrderedDict
-from typing import FrozenSet, List, Optional
+from typing import List, Optional
 
 from . import macro_collector
 
@@ -23,8 +23,13 @@ def __init__(self) -> None:
     def remove_unwanted_macros(
             constructors: macro_collector.PSAMacroEnumerator
     ) -> None:
-        # Mbed TLS does not support finite-field DSA.
+        """Remove constructors that should be exckuded from systematic testing."""
+        # Mbed TLS does not support finite-field DSA, but 3.6 defines DSA
+        # identifiers for historical reasons.
         # Don't attempt to generate any related test case.
+        # The corresponding test cases would be commented out anyway,
+        # but for DSA, we don't have enough support in the test scripts
+        # to generate these test cases.
         constructors.key_types.discard('PSA_KEY_TYPE_DSA_KEY_PAIR')
         constructors.key_types.discard('PSA_KEY_TYPE_DSA_PUBLIC_KEY')
 
@@ -52,10 +57,16 @@ def read_psa_interface(self) -> macro_collector.PSAMacroEnumerator:
         return constructors
 
 
-def psa_want_symbol(name: str) -> str:
-    """Return the PSA_WANT_xxx symbol associated with a PSA crypto feature."""
+def psa_want_symbol(name: str, prefix: Optional[str] = None) -> str:
+    """Return the PSA_WANT_xxx symbol associated with a PSA crypto feature.
+
+    You can use an altenative `prefix`, e.g. 'MBEDTLS_PSA_BUILTIN_'
+    when specifically testing builtin implementations.
+    """
+    if prefix is None:
+        prefix = 'PSA_WANT_'
     if name.startswith('PSA_'):
-        return name[:4] + 'WANT_' + name[4:]
+        return prefix + name[4:]
     else:
         raise ValueError('Unable to determine the PSA_WANT_ symbol for ' + name)
 
@@ -83,18 +94,23 @@ def finish_family_dependencies(dependencies: List[str], bits: int) -> List[str]:
     'PSA_ALG_KEY_AGREEMENT', # chaining
     'PSA_ALG_TRUNCATED_MAC', # modifier
 ])
-def automatic_dependencies(*expressions: str) -> List[str]:
+def automatic_dependencies(*expressions: str,
+                           prefix: Optional[str] = None) -> List[str]:
     """Infer dependencies of a test case by looking for PSA_xxx symbols.
 
     The arguments are strings which should be C expressions. Do not use
     string literals or comments as this function is not smart enough to
     skip them.
+
+    `prefix`: prefix to use in dependencies. Defaults to ``'PSA_WANT_'``.
+              Use ``'MBEDTLS_PSA_BUILTIN_'`` when specifically testing
+              builtin implementations.
     """
     used = set()
     for expr in expressions:
         used.update(re.findall(r'PSA_(?:ALG|ECC_FAMILY|DH_FAMILY|KEY_TYPE)_\w+', expr))
     used.difference_update(SYMBOLS_WITHOUT_DEPENDENCY)
-    return sorted(psa_want_symbol(name) for name in used)
+    return sorted(psa_want_symbol(name, prefix=prefix) for name in used)
 
 # Define set of regular expressions and dependencies to optionally append
 # extra dependencies for test case based on key description.
@@ -123,38 +139,6 @@ def generate_deps_from_description(
 
     return dep_list
 
-# A temporary hack: at the time of writing, not all dependency symbols
-# are implemented yet. Skip test cases for which the dependency symbols are
-# not available. Once all dependency symbols are available, this hack must
-# be removed so that a bug in the dependency symbols properly leads to a test
-# failure.
-def read_implemented_dependencies(filename: str) -> FrozenSet[str]:
-    return frozenset(symbol
-                     for line in open(filename)
-                     for symbol in re.findall(r'\bPSA_WANT_\w+\b', line))
-_implemented_dependencies = None #type: Optional[FrozenSet[str]] #pylint: disable=invalid-name
-def hack_dependencies_not_implemented(dependencies: List[str]) -> None:
-    """
-    Hack dependencies to skip test cases for which at least one dependency
-    symbol is not available yet.
-    """
-    global _implemented_dependencies #pylint: disable=global-statement,invalid-name
-    if _implemented_dependencies is None:
-        # Temporary, while Mbed TLS does not just rely on the TF-PSA-Crypto
-        # build system to build its crypto library. When it does, the first
-        # case can just be removed.
-        if os.path.isdir('tf-psa-crypto'):
-            _implemented_dependencies = \
-                read_implemented_dependencies('tf-psa-crypto/include/psa/crypto_config.h')
-        else:
-            _implemented_dependencies = \
-                read_implemented_dependencies('include/psa/crypto_config.h')
-
-    if not all((dep.lstrip('!') in _implemented_dependencies or
-                not dep.lstrip('!').startswith('PSA_WANT'))
-               for dep in dependencies):
-        dependencies.append('DEPENDENCY_NOT_IMPLEMENTED_YET')
-
 def tweak_key_pair_dependency(dep: str, usage: str):
     """
     This helper function add the proper suffix to PSA_WANT_KEY_TYPE_xxx_KEY_PAIR