#!/usr/bin/env python3 """Analyze the test outcomes from a full CI run. This script can also run on outcomes from a partial run, but the results are less likely to be useful. """ import argparse import sys import traceback import re import subprocess import os import typing import check_test_cases # `ComponentOutcomes` is a named tuple which is defined as: # ComponentOutcomes( # successes = { # "", # ... # }, # failures = { # "", # ... # } # ) # suite_case = ";" ComponentOutcomes = typing.NamedTuple('ComponentOutcomes', [('successes', typing.Set[str]), ('failures', typing.Set[str])]) # `Outcomes` is a representation of the outcomes file, # which defined as: # Outcomes = { # "": ComponentOutcomes, # ... # } Outcomes = typing.Dict[str, ComponentOutcomes] class Results: """Process analysis results.""" def __init__(self): self.error_count = 0 self.warning_count = 0 def new_section(self, fmt, *args, **kwargs): self._print_line('\n*** ' + fmt + ' ***\n', *args, **kwargs) def info(self, fmt, *args, **kwargs): self._print_line('Info: ' + fmt, *args, **kwargs) def error(self, fmt, *args, **kwargs): self.error_count += 1 self._print_line('Error: ' + fmt, *args, **kwargs) def warning(self, fmt, *args, **kwargs): self.warning_count += 1 self._print_line('Warning: ' + fmt, *args, **kwargs) @staticmethod def _print_line(fmt, *args, **kwargs): sys.stderr.write((fmt + '\n').format(*args, **kwargs)) def execute_reference_driver_tests(results: Results, ref_component: str, driver_component: str, \ outcome_file: str) -> None: """Run the tests specified in ref_component and driver_component. Results are stored in the output_file and they will be used for the following coverage analysis""" results.new_section("Test {} and {}", ref_component, driver_component) shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \ " " + ref_component + " " + driver_component results.info("Running: {}", shell_command) ret_val = subprocess.run(shell_command.split(), check=False).returncode if ret_val != 0: results.error("failed to run reference/driver components") def analyze_coverage(results: Results, outcomes: Outcomes, allow_list: typing.List[str], full_coverage: bool) -> None: """Check that all available test cases are executed at least once.""" available = check_test_cases.collect_available_test_cases() for suite_case in available: hit = any(suite_case in comp_outcomes.successes or suite_case in comp_outcomes.failures for comp_outcomes in outcomes.values()) if not hit and suite_case not in allow_list: if full_coverage: results.error('Test case not executed: {}', suite_case) else: results.warning('Test case not executed: {}', suite_case) elif hit and suite_case in allow_list: # Test Case should be removed from the allow list. if full_coverage: results.error('Allow listed test case was executed: {}', suite_case) else: results.warning('Allow listed test case was executed: {}', suite_case) def name_matches_pattern(name: str, str_or_re) -> bool: """Check if name matches a pattern, that may be a string or regex. - If the pattern is a string, name must be equal to match. - If the pattern is a regex, name must fully match. """ # The CI's python is too old for re.Pattern #if isinstance(str_or_re, re.Pattern): if not isinstance(str_or_re, str): return str_or_re.fullmatch(name) is not None else: return str_or_re == name def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, component_ref: str, component_driver: str, ignored_suites: typing.List[str], ignored_tests=None) -> None: """Check that all tests passing in the reference component are also passing in the corresponding driver component. Skip: - full test suites provided in ignored_suites list - only some specific test inside a test suite, for which the corresponding output string is provided """ ref_outcomes = outcomes.get("component_" + component_ref) driver_outcomes = outcomes.get("component_" + component_driver) if ref_outcomes is None or driver_outcomes is None: results.error("required components are missing: bad outcome file?") return if not ref_outcomes.successes: results.error("no passing test in reference component: bad outcome file?") return for suite_case in ref_outcomes.successes: # suite_case is like "test_suite_foo.bar;Description of test case" (full_test_suite, test_string) = suite_case.split(';') test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name # Immediately skip fully-ignored test suites if test_suite in ignored_suites or full_test_suite in ignored_suites: continue # For ignored test cases inside test suites, just remember and: # don't issue an error if they're skipped with drivers, # but issue an error if they're not (means we have a bad entry). ignored = False if full_test_suite in ignored_tests: for str_or_re in ignored_tests[full_test_suite]: if name_matches_pattern(test_string, str_or_re): ignored = True if not ignored and not suite_case in driver_outcomes.successes: results.error("PASS -> SKIP/FAIL: {}", suite_case) if ignored and suite_case in driver_outcomes.successes: results.error("uselessly ignored: {}", suite_case) def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None: """Run all analyses on the given outcome collection.""" analyze_coverage(results, outcomes, args['allow_list'], args['full_coverage']) def read_outcome_file(outcome_file: str) -> Outcomes: """Parse an outcome file and return an outcome collection. """ outcomes = {} with open(outcome_file, 'r', encoding='utf-8') as input_file: for line in input_file: (_platform, component, suite, case, result, _cause) = line.split(';') # Note that `component` is not unique. If a test case passes on Linux # and fails on FreeBSD, it'll end up in both the successes set and # the failures set. suite_case = ';'.join([suite, case]) if component not in outcomes: outcomes[component] = ComponentOutcomes(set(), set()) if result == 'PASS': outcomes[component].successes.add(suite_case) elif result == 'FAIL': outcomes[component].failures.add(suite_case) return outcomes def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None: """Perform coverage analysis.""" results.new_section("Analyze coverage") analyze_outcomes(results, outcomes, args) def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None: """Perform driver vs reference analyze.""" results.new_section("Analyze driver {} vs reference {}", args['component_driver'], args['component_ref']) ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] analyze_driver_vs_reference(results, outcomes, args['component_ref'], args['component_driver'], ignored_suites, args['ignored_tests']) # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS = { 'analyze_coverage': { 'test_function': do_analyze_coverage, 'args': { 'allow_list': [ # Algorithm not supported yet 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', # Algorithm not supported yet 'test_suite_psa_crypto_metadata;Cipher: XTS', ], 'full_coverage': False, } }, # There are 2 options to use analyze_driver_vs_reference_xxx locally: # 1. Run tests and then analysis: # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx # 2. Let this script run both automatically: # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx 'analyze_driver_vs_reference_hash': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', 'ignored_suites': [ 'shax', 'mdx', # the software implementations that are being excluded 'md.psa', # purposefully depends on whether drivers are present 'psa_crypto_low_hash.generated', # testing the builtins ], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], } } }, 'analyze_driver_vs_reference_hmac': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_hmac', 'component_driver': 'test_psa_crypto_config_accel_hmac', 'ignored_suites': [ # These suites require legacy hash support, which is disabled # in the accelerated component. 'shax', 'mdx', # This suite tests builtins directly, but these are missing # in the accelerated case. 'psa_crypto_low_hash.generated', ], 'ignored_tests': { 'test_suite_md': [ # Builtin HMAC is not supported in the accelerate component. re.compile('.*HMAC.*'), # Following tests make use of functions which are not available # when MD_C is disabled, as it happens in the accelerated # test component. re.compile('generic .* Hash file .*'), 'MD list', ], 'test_suite_md.psa': [ # "legacy only" tests require hash algorithms to be NOT # accelerated, but this of course false for the accelerated # test component. re.compile('PSA dispatch .* legacy only'), ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], } } }, 'analyze_driver_vs_reference_cipher_aead_cmac': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_cipher_aead_cmac', 'component_driver': 'test_psa_crypto_config_accel_cipher_aead_cmac', # Modules replaced by drivers. 'ignored_suites': [ # low-level (block/stream) cipher modules 'aes', 'aria', 'camellia', 'des', 'chacha20', # AEAD modes and CMAC 'ccm', 'chachapoly', 'cmac', 'gcm', # The Cipher abstraction layer 'cipher', ], 'ignored_tests': { # PEM decryption is not supported so far. # The rest of PEM (write, unencrypted read) works though. 'test_suite_pem': [ re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], # Following tests depend on AES_C/DES_C but are not about # them really, just need to know some error code is there. 'test_suite_error': [ 'Low and high error', 'Single low error' ], # Similar to test_suite_error above. 'test_suite_version': [ 'Check for MBEDTLS_AES_C when already present', ], # The en/decryption part of PKCS#12 is not supported so far. # The rest of PKCS#12 (key derivation) works though. 'test_suite_pkcs12': [ re.compile(r'PBE Encrypt, .*'), re.compile(r'PBE Decrypt, .*'), ], # The en/decryption part of PKCS#5 is not supported so far. # The rest of PKCS#5 (PBKDF2) works though. 'test_suite_pkcs5': [ re.compile(r'PBES2 Encrypt, .*'), re.compile(r'PBES2 Decrypt .*'), ], # Encrypted keys are not supported so far. # pylint: disable=line-too-long 'test_suite_pkparse': [ 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'), ], } } }, 'analyze_driver_vs_reference_ecp_light_only': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', 'ignored_suites': [ # Modules replaced by drivers 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], # This test wants a legacy function that takes f_rng, p_rng # arguments, and uses legacy ECDSA for that. The test is # really about the wrapper around the PSA RNG, not ECDSA. 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], # In the accelerated test ECP_C is not set (only ECP_LIGHT is) # so we must ignore disparities in the tests for which ECP_C # is required. 'test_suite_ecp': [ re.compile(r'ECP check public-private .*'), re.compile(r'ECP calculate public: .*'), re.compile(r'ECP gen keypair .*'), re.compile(r'ECP point muladd .*'), re.compile(r'ECP point multiplication .*'), re.compile(r'ECP test vectors .*'), ], 'test_suite_ssl': [ # This deprecated function is only present when ECP_C is On. 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], } } }, 'analyze_driver_vs_reference_no_ecp_at_all': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', 'ignored_suites': [ # Modules replaced by drivers 'ecp', 'ecdsa', 'ecdh', 'ecjpake', ], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], 'test_suite_pkparse': [ # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED # is automatically enabled in build_info.h (backward compatibility) # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a # consequence compressed points are supported in the reference # component but not in the accelerated one, so they should be skipped # while checking driver's coverage. re.compile(r'Parse EC Key .*compressed\)'), re.compile(r'Parse Public EC Key .*compressed\)'), ], # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], } } }, 'analyze_driver_vs_reference_ecc_no_bignum': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', 'ignored_suites': [ # Modules replaced by drivers 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], # See no_ecp_at_all 'test_suite_pkparse': [ re.compile(r'Parse EC Key .*compressed\)'), re.compile(r'Parse Public EC Key .*compressed\)'), ], 'test_suite_asn1parse': [ 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ re.compile(r'ASN.1 Write mpi.*'), ], 'test_suite_debug': [ re.compile(r'Debug print mbedtls_mpi.*'), ], # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], } } }, 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', 'ignored_suites': [ # Modules replaced by drivers 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], # See no_ecp_at_all 'test_suite_pkparse': [ re.compile(r'Parse EC Key .*compressed\)'), re.compile(r'Parse Public EC Key .*compressed\)'), ], 'test_suite_asn1parse': [ 'INTEGER too large for mpi', ], 'test_suite_asn1write': [ re.compile(r'ASN.1 Write mpi.*'), ], 'test_suite_debug': [ re.compile(r'Debug print mbedtls_mpi.*'), ], # See ecp_light_only 'test_suite_ssl': [ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', ], } } }, 'analyze_driver_vs_reference_ffdh_alg': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_ffdh', 'component_driver': 'test_psa_crypto_config_accel_ffdh', 'ignored_suites': ['dhm'], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], } } }, 'analyze_driver_vs_reference_tfm_config': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_tfm_config', 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', 'ignored_suites': [ # Modules replaced by drivers 'asn1parse', 'asn1write', 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 'bignum.generated', 'bignum.misc', ], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], # See ecp_light_only 'test_suite_random': [ 'PSA classic wrapper: ECDSA signature (SECP256R1)', ], } } }, 'analyze_driver_vs_reference_rsa': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_psa_crypto_config_reference_rsa_crypto', 'component_driver': 'test_psa_crypto_config_accel_rsa_crypto', 'ignored_suites': [ # Modules replaced by drivers. 'rsa', 'pkcs1_v15', 'pkcs1_v21', # We temporarily don't care about PK stuff. 'pk', 'pkwrite', 'pkparse' ], 'ignored_tests': { 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], # Following tests depend on RSA_C but are not about # them really, just need to know some error code is there. 'test_suite_error': [ 'Low and high error', 'Single high error' ], # Constant time operations only used for PKCS1_V15 'test_suite_constant_time': [ re.compile(r'mbedtls_ct_zeroize_if .*'), re.compile(r'mbedtls_ct_memmove_left .*') ], 'test_suite_psa_crypto': [ # We don't support generate_key_ext entry points # in drivers yet. re.compile(r'PSA generate key ext: RSA, e=.*'), ], } } }, 'analyze_block_cipher_dispatch': { 'test_function': do_analyze_driver_vs_reference, 'args': { 'component_ref': 'test_full_block_cipher_legacy_dispatch', 'component_driver': 'test_full_block_cipher_psa_dispatch', 'ignored_suites': [ # Skipped in the accelerated component 'aes', 'aria', 'camellia', # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in # order for the cipher module (actually cipher_wrapper) to work # properly. However these symbols are disabled in the accelerated # component so we ignore them. 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria', 'cipher.camellia', ], 'ignored_tests': { 'test_suite_cmac': [ # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, # but these are not available in the accelerated component. 'CMAC null arguments', re.compile('CMAC.* (AES|ARIA|Camellia).*'), ], 'test_suite_cipher.padding': [ # Following tests require AES_C/CAMELLIA_C to be enabled, # but these are not available in the accelerated component. re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'), ], 'test_suite_pkcs5': [ # The AES part of PKCS#5 PBES2 is not yet supported. # The rest of PKCS#5 (PBKDF2) works, though. re.compile(r'PBES2 .* AES-.*') ], 'test_suite_pkparse': [ # PEM (called by pkparse) requires AES_C in order to decrypt # the key, but this is not available in the accelerated # component. re.compile('Parse RSA Key.*(password|AES-).*'), ], 'test_suite_pem': [ # Following tests require AES_C, but this is diabled in the # accelerated component. re.compile('PEM read .*AES.*'), 'PEM read (unknown encryption algorithm)', ], 'test_suite_error': [ # Following tests depend on AES_C but are not about them # really, just need to know some error code is there. 'Single low error', 'Low and high error', ], 'test_suite_version': [ # Similar to test_suite_error above. 'Check for MBEDTLS_AES_C when already present', ], 'test_suite_platform': [ # Incompatible with sanitizers (e.g. ASan). If the driver # component uses a sanitizer but the reference component # doesn't, we have a PASS vs SKIP mismatch. 'Check mbedtls_calloc overallocation', ], } } } } def main(): main_results = Results() try: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('outcomes', metavar='OUTCOMES.CSV', help='Outcome file to analyze') parser.add_argument('specified_tasks', default='all', nargs='?', help='Analysis to be done. By default, run all tasks. ' 'With one or more TASK, run only those. ' 'TASK can be the name of a single task or ' 'comma/space-separated list of tasks. ') parser.add_argument('--list', action='store_true', help='List all available tasks and exit.') parser.add_argument('--require-full-coverage', action='store_true', dest='full_coverage', help="Require all available " "test cases to be executed and issue an error " "otherwise. This flag is ignored if 'task' is " "neither 'all' nor 'analyze_coverage'") options = parser.parse_args() if options.list: for task in KNOWN_TASKS: print(task) sys.exit(0) if options.specified_tasks == 'all': tasks_list = KNOWN_TASKS.keys() else: tasks_list = re.split(r'[, ]+', options.specified_tasks) for task in tasks_list: if task not in KNOWN_TASKS: sys.stderr.write('invalid task: {}\n'.format(task)) sys.exit(2) KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage # If the outcome file exists, parse it once and share the result # among tasks to improve performance. # Otherwise, it will be generated by execute_reference_driver_tests. if not os.path.exists(options.outcomes): if len(tasks_list) > 1: sys.stderr.write("mutiple tasks found, please provide a valid outcomes file.\n") sys.exit(2) task_name = tasks_list[0] task = KNOWN_TASKS[task_name] if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name)) sys.exit(2) execute_reference_driver_tests(main_results, task['args']['component_ref'], task['args']['component_driver'], options.outcomes) outcomes = read_outcome_file(options.outcomes) for task in tasks_list: test_function = KNOWN_TASKS[task]['test_function'] test_args = KNOWN_TASKS[task]['args'] test_function(main_results, outcomes, test_args) main_results.info("Overall results: {} warnings and {} errors", main_results.warning_count, main_results.error_count) sys.exit(0 if (main_results.error_count == 0) else 1) except Exception: # pylint: disable=broad-except # Print the backtrace and exit explicitly with our chosen status. traceback.print_exc() sys.exit(120) if __name__ == '__main__': main()