Skip to content

Commit

Permalink
Driver vs referenee: ignore relevant configuration differences
Browse files Browse the repository at this point in the history 
The driver-vs-reference checks compare test results in different
configurations. Ignore the test results that report differences in
configurations that were the point of the comparison. Do compare other
configuration reports: this will let us know if the configurations diverge
in an unexpected way.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
  • Loading branch information
@gilles-peskine-arm
gilles-peskine-arm committed May 24, 2024
1 parent c10076e commit 3b40344
Showing 1 changed file with 56 additions and 4 deletions.
60 changes: 56 additions & 4 deletions tests/scripts/analyze_outcomes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,10 @@ def analyze_driver_vs_reference(results: Results, outcomes: Outcomes,
# don't issue an error if they're skipped with drivers,
# but issue an error if they're not (means we have a bad entry).
ignored = False
if full_test_suite in ignored_tests:
for str_or_re in ignored_tests[full_test_suite]:
if name_matches_pattern(test_string, str_or_re):
ignored = True
for str_or_re in (ignored_tests.get(full_test_suite, []) +
ignored_tests.get(test_suite, [])):
if name_matches_pattern(test_string, str_or_re):
ignored = True

if not ignored and not suite_case in driver_outcomes.successes:
results.error("PASS -> SKIP/FAIL: {}", suite_case)
Expand Down Expand Up @@ -242,6 +242,9 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'psa_crypto_low_hash.generated', # testing the builtins
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand All @@ -265,6 +268,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'psa_crypto_low_hash.generated',
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
re.compile(r'.*\bMBEDTLS_MD_C\b')
],
'test_suite_md': [
# Builtin HMAC is not supported in the accelerate component.
re.compile('.*HMAC.*'),
Expand Down Expand Up @@ -304,6 +311,12 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'cipher',
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'),
re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'),
re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
re.compile(r'.*\bMBEDTLS_CIPHER_.*'),
],
# PEM decryption is not supported so far.
# The rest of PEM (write, unencrypted read) works though.
'test_suite_pem': [
Expand Down Expand Up @@ -357,6 +370,9 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'ecdsa', 'ecdh', 'ecjpake',
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand Down Expand Up @@ -397,6 +413,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'ecp', 'ecdsa', 'ecdh', 'ecjpake',
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand Down Expand Up @@ -436,6 +456,11 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'bignum.generated', 'bignum.misc',
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand Down Expand Up @@ -485,6 +510,13 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
# provide), even with MBEDTLS_USE_PSA_CRYPTO.
re.compile(r'PSK callback:.*\bdhe-psk\b.*'),
],
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'),
re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand Down Expand Up @@ -523,6 +555,9 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'component_driver': 'test_psa_crypto_config_accel_ffdh',
'ignored_suites': ['dhm'],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand All @@ -545,6 +580,15 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'bignum.generated', 'bignum.misc',
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'),
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'),
re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*')
],
'test_suite_config.crypto_combinations': [
'Config: ECC: Weierstrass curves only',
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand All @@ -570,6 +614,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'pk', 'pkwrite', 'pkparse'
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'),
re.compile(r'.*\bMBEDTLS_GENPRIME\b.*')
],
'test_suite_platform': [
# Incompatible with sanitizers (e.g. ASan). If the driver
# component uses a sanitizer but the reference component
Expand Down Expand Up @@ -611,6 +659,10 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
'cipher.camellia',
],
'ignored_tests': {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'),
re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
],
'test_suite_cmac': [
# Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled,
# but these are not available in the accelerated component.
Expand Down

0 comments on commit 3b40344

Please sign in to comment.