"""Collect macro definitions from header files. """ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # import itertools import re from typing import Dict, IO, Iterable, Iterator, List, Optional, Pattern, Set, Tuple, Union class ReadFileLineException(Exception): def __init__(self, filename: str, line_number: Union[int, str]) -> None: message = 'in {} at {}'.format(filename, line_number) super(ReadFileLineException, self).__init__(message) self.filename = filename self.line_number = line_number class read_file_lines: # Dear Pylint, conventionally, a context manager class name is lowercase. # pylint: disable=invalid-name,too-few-public-methods """Context manager to read a text file line by line. ``` with read_file_lines(filename) as lines: for line in lines: process(line) ``` is equivalent to ``` with open(filename, 'r') as input_file: for line in input_file: process(line) ``` except that if process(line) raises an exception, then the read_file_lines snippet annotates the exception with the file name and line number. """ def __init__(self, filename: str, binary: bool = False) -> None: self.filename = filename self.file = None #type: Optional[IO[str]] self.line_number = 'entry' #type: Union[int, str] self.generator = None #type: Optional[Iterable[Tuple[int, str]]] self.binary = binary def __enter__(self) -> 'read_file_lines': self.file = open(self.filename, 'rb' if self.binary else 'r') self.generator = enumerate(self.file) return self def __iter__(self) -> Iterator[str]: assert self.generator is not None for line_number, content in self.generator: self.line_number = line_number yield content self.line_number = 'exit' def __exit__(self, exc_type, exc_value, exc_traceback) -> None: if self.file is not None: self.file.close() if exc_type is not None: raise ReadFileLineException(self.filename, self.line_number) \ from exc_value class PSAMacroEnumerator: """Information about constructors of various PSA Crypto types. This includes macro names as well as information about their arguments when applicable. This class only provides ways to enumerate expressions that evaluate to values of the covered types. Derived classes are expected to populate the set of known constructors of each kind, as well as populate `self.arguments_for` for arguments that are not of a kind that is enumerated here. """ #pylint: disable=too-many-instance-attributes def __init__(self) -> None: """Set up an empty set of known constructor macros. """ self.statuses = set() #type: Set[str] self.lifetimes = set() #type: Set[str] self.locations = set() #type: Set[str] self.persistence_levels = set() #type: Set[str] self.algorithms = set() #type: Set[str] self.ecc_curves = set() #type: Set[str] self.dh_groups = set() #type: Set[str] self.key_types = set() #type: Set[str] self.key_usage_flags = set() #type: Set[str] self.hash_algorithms = set() #type: Set[str] self.mac_algorithms = set() #type: Set[str] self.ka_algorithms = set() #type: Set[str] self.kdf_algorithms = set() #type: Set[str] self.pake_algorithms = set() #type: Set[str] self.aead_algorithms = set() #type: Set[str] self.sign_algorithms = set() #type: Set[str] # macro name -> list of argument names self.argspecs = {} #type: Dict[str, List[str]] # argument name -> list of values self.arguments_for = { 'mac_length': [], 'min_mac_length': [], 'tag_length': [], 'min_tag_length': [], } #type: Dict[str, List[str]] # Whether to include intermediate macros in enumerations. Intermediate # macros serve as category headers and are not valid values of their # type. See `is_internal_name`. # Always false in this class, may be set to true in derived classes. self.include_intermediate = False def is_internal_name(self, name: str) -> bool: """Whether this is an internal macro. Internal macros will be skipped.""" if not self.include_intermediate: if name.endswith('_BASE') or name.endswith('_NONE'): return True if '_CATEGORY_' in name: return True return name.endswith('_FLAG') or name.endswith('_MASK') def gather_arguments(self) -> None: """Populate the list of values for macro arguments. Call this after parsing all the inputs. """ self.arguments_for['hash_alg'] = sorted(self.hash_algorithms) self.arguments_for['mac_alg'] = sorted(self.mac_algorithms) self.arguments_for['ka_alg'] = sorted(self.ka_algorithms) self.arguments_for['kdf_alg'] = sorted(self.kdf_algorithms) self.arguments_for['aead_alg'] = sorted(self.aead_algorithms) self.arguments_for['sign_alg'] = sorted(self.sign_algorithms) self.arguments_for['curve'] = sorted(self.ecc_curves) self.arguments_for['group'] = sorted(self.dh_groups) self.arguments_for['persistence'] = sorted(self.persistence_levels) self.arguments_for['location'] = sorted(self.locations) self.arguments_for['lifetime'] = sorted(self.lifetimes) @staticmethod def _format_arguments(name: str, arguments: Iterable[str]) -> str: """Format a macro call with arguments. The resulting format is consistent with `InputsForTest.normalize_argument`. """ return name + '(' + ', '.join(arguments) + ')' _argument_split_re = re.compile(r' *, *') @classmethod def _argument_split(cls, arguments: str) -> List[str]: return re.split(cls._argument_split_re, arguments) def distribute_arguments(self, name: str) -> Iterator[str]: """Generate macro calls with each tested argument set. If name is a macro without arguments, just yield "name". If name is a macro with arguments, yield a series of "name(arg1,...,argN)" where each argument takes each possible value at least once. """ try: if name not in self.argspecs: yield name return argspec = self.argspecs[name] if argspec == []: yield name + '()' return argument_lists = [self.arguments_for[arg] for arg in argspec] arguments = [values[0] for values in argument_lists] yield self._format_arguments(name, arguments) # Dear Pylint, enumerate won't work here since we're modifying # the array. # pylint: disable=consider-using-enumerate for i in range(len(arguments)): for value in argument_lists[i][1:]: arguments[i] = value yield self._format_arguments(name, arguments) arguments[i] = argument_lists[i][0] except BaseException as e: raise Exception('distribute_arguments({})'.format(name)) from e def distribute_arguments_without_duplicates( self, seen: Set[str], name: str ) -> Iterator[str]: """Same as `distribute_arguments`, but don't repeat seen results.""" for result in self.distribute_arguments(name): if result not in seen: seen.add(result) yield result def generate_expressions(self, names: Iterable[str]) -> Iterator[str]: """Generate expressions covering values constructed from the given names. `names` can be any iterable collection of macro names. For example: * ``generate_expressions(['PSA_ALG_CMAC', 'PSA_ALG_HMAC'])`` generates ``'PSA_ALG_CMAC'`` as well as ``'PSA_ALG_HMAC(h)'`` for every known hash algorithm ``h``. * ``macros.generate_expressions(macros.key_types)`` generates all key types. """ seen = set() #type: Set[str] return itertools.chain(*( self.distribute_arguments_without_duplicates(seen, name) for name in names )) class PSAMacroCollector(PSAMacroEnumerator): """Collect PSA crypto macro definitions from C header files. """ def __init__(self, include_intermediate: bool = False) -> None: """Set up an object to collect PSA macro definitions. Call the read_file method of the constructed object on each header file. * include_intermediate: if true, include intermediate macros such as PSA_XXX_BASE that do not designate semantic values. """ super().__init__() self.include_intermediate = include_intermediate self.key_types_from_curve = {} #type: Dict[str, str] self.key_types_from_group = {} #type: Dict[str, str] self.algorithms_from_hash = {} #type: Dict[str, str] @staticmethod def algorithm_tester(name: str) -> str: """The predicate for whether an algorithm is built from the given constructor. The given name must be the name of an algorithm constructor of the form ``PSA_ALG_xxx`` which is used as ``PSA_ALG_xxx(yyy)`` to build an algorithm value. Return the corresponding predicate macro which is used as ``predicate(alg)`` to test whether ``alg`` can be built as ``PSA_ALG_xxx(yyy)``. The predicate is usually called ``PSA_ALG_IS_xxx``. """ prefix = 'PSA_ALG_' assert name.startswith(prefix) midfix = 'IS_' suffix = name[len(prefix):] if suffix in ['DSA', 'ECDSA']: midfix += 'RANDOMIZED_' elif suffix == 'RSA_PSS': suffix += '_STANDARD_SALT' return prefix + midfix + suffix def record_algorithm_subtype(self, name: str, expansion: str) -> None: """Record the subtype of an algorithm constructor. Given a ``PSA_ALG_xxx`` macro name and its expansion, if the algorithm is of a subtype that is tracked in its own set, add it to the relevant set. """ # This code is very ad hoc and fragile. It should be replaced by # something more robust. if re.match(r'MAC(?:_|\Z)', name): self.mac_algorithms.add(name) elif re.match(r'KDF(?:_|\Z)', name): self.kdf_algorithms.add(name) elif re.search(r'0x020000[0-9A-Fa-f]{2}', expansion): self.hash_algorithms.add(name) elif re.search(r'0x03[0-9A-Fa-f]{6}', expansion): self.mac_algorithms.add(name) elif re.search(r'0x05[0-9A-Fa-f]{6}', expansion): self.aead_algorithms.add(name) elif re.search(r'0x09[0-9A-Fa-f]{2}0000', expansion): self.ka_algorithms.add(name) elif re.search(r'0x08[0-9A-Fa-f]{6}', expansion): self.kdf_algorithms.add(name) # "#define" followed by a macro name with either no parameters # or a single parameter and a non-empty expansion. # Grab the macro name in group 1, the parameter name if any in group 2 # and the expansion in group 3. _define_directive_re = re.compile(r'\s*#\s*define\s+(\w+)' + r'(?:\s+|\((\w+)\)\s*)' + r'(.+)') _deprecated_definition_re = re.compile(r'\s*MBEDTLS_DEPRECATED') def read_line(self, line): """Parse a C header line and record the PSA identifier it defines if any. This function analyzes lines that start with "#define PSA_" (up to non-significant whitespace) and skips all non-matching lines. """ # pylint: disable=too-many-branches m = re.match(self._define_directive_re, line) if not m: return name, parameter, expansion = m.groups() expansion = re.sub(r'/\*.*?\*/|//.*', r' ', expansion) if parameter: self.argspecs[name] = [parameter] if re.match(self._deprecated_definition_re, expansion): # Skip deprecated values, which are assumed to be # backward compatibility aliases that share # numerical values with non-deprecated values. return if self.is_internal_name(name): # Macro only to build actual values return elif (name.startswith('PSA_ERROR_') or name == 'PSA_SUCCESS') \ and not parameter: self.statuses.add(name) elif name.startswith('PSA_KEY_TYPE_') and not parameter: self.key_types.add(name) elif name.startswith('PSA_KEY_TYPE_') and parameter == 'curve': self.key_types_from_curve[name] = name[:13] + 'IS_' + name[13:] elif name.startswith('PSA_KEY_TYPE_') and parameter == 'group': self.key_types_from_group[name] = name[:13] + 'IS_' + name[13:] elif name.startswith('PSA_ECC_FAMILY_') and not parameter: self.ecc_curves.add(name) elif name.startswith('PSA_DH_FAMILY_') and not parameter: self.dh_groups.add(name) elif name.startswith('PSA_ALG_') and not parameter: if name in ['PSA_ALG_ECDSA_BASE', 'PSA_ALG_RSA_PKCS1V15_SIGN_BASE']: # Ad hoc skipping of duplicate names for some numerical values return self.algorithms.add(name) self.record_algorithm_subtype(name, expansion) elif name.startswith('PSA_ALG_') and parameter == 'hash_alg': self.algorithms_from_hash[name] = self.algorithm_tester(name) elif name.startswith('PSA_KEY_USAGE_') and not parameter: self.key_usage_flags.add(name) else: # Other macro without parameter return _nonascii_re = re.compile(rb'[^\x00-\x7f]+') _continued_line_re = re.compile(rb'\\\r?\n\Z') def read_file(self, header_file): for line in header_file: m = re.search(self._continued_line_re, line) while m: cont = next(header_file) line = line[:m.start(0)] + cont m = re.search(self._continued_line_re, line) line = re.sub(self._nonascii_re, rb'', line).decode('ascii') self.read_line(line) class InputsForTest(PSAMacroEnumerator): # pylint: disable=too-many-instance-attributes """Accumulate information about macros to test. enumerate This includes macro names as well as information about their arguments when applicable. """ def __init__(self) -> None: super().__init__() self.all_declared = set() #type: Set[str] # Identifier prefixes self.table_by_prefix = { 'ERROR': self.statuses, 'ALG': self.algorithms, 'ECC_CURVE': self.ecc_curves, 'DH_GROUP': self.dh_groups, 'KEY_LIFETIME': self.lifetimes, 'KEY_LOCATION': self.locations, 'KEY_PERSISTENCE': self.persistence_levels, 'KEY_TYPE': self.key_types, 'KEY_USAGE': self.key_usage_flags, } #type: Dict[str, Set[str]] # Test functions self.table_by_test_function = { # Any function ending in _algorithm also gets added to # self.algorithms. 'key_type': [self.key_types], 'block_cipher_key_type': [self.key_types], 'stream_cipher_key_type': [self.key_types], 'ecc_key_family': [self.ecc_curves], 'ecc_key_types': [self.ecc_curves], 'dh_key_family': [self.dh_groups], 'dh_key_types': [self.dh_groups], 'hash_algorithm': [self.hash_algorithms], 'mac_algorithm': [self.mac_algorithms], 'cipher_algorithm': [], 'hmac_algorithm': [self.mac_algorithms, self.sign_algorithms], 'aead_algorithm': [self.aead_algorithms], 'key_derivation_algorithm': [self.kdf_algorithms], 'key_agreement_algorithm': [self.ka_algorithms], 'asymmetric_signature_algorithm': [self.sign_algorithms], 'asymmetric_signature_wildcard': [self.algorithms], 'asymmetric_encryption_algorithm': [], 'pake_algorithm': [self.pake_algorithms], 'other_algorithm': [], 'lifetime': [self.lifetimes], } #type: Dict[str, List[Set[str]]] mac_lengths = [str(n) for n in [ 1, # minimum expressible 4, # minimum allowed by policy 13, # an odd size in a plausible range 14, # an even non-power-of-two size in a plausible range 16, # same as full size for at least one algorithm 63, # maximum expressible ]] self.arguments_for['mac_length'] += mac_lengths self.arguments_for['min_mac_length'] += mac_lengths aead_lengths = [str(n) for n in [ 1, # minimum expressible 4, # minimum allowed by policy 13, # an odd size in a plausible range 14, # an even non-power-of-two size in a plausible range 16, # same as full size for at least one algorithm 63, # maximum expressible ]] self.arguments_for['tag_length'] += aead_lengths self.arguments_for['min_tag_length'] += aead_lengths def add_numerical_values(self) -> None: """Add numerical values that are not supported to the known identifiers.""" # Sets of names per type self.algorithms.add('0xffffffff') self.ecc_curves.add('0xff') self.dh_groups.add('0xff') self.key_types.add('0xffff') self.key_usage_flags.add('0x80000000') # Hard-coded values for unknown algorithms # # These have to have values that are correct for their respective # PSA_ALG_IS_xxx macros, but are also not currently assigned and are # not likely to be assigned in the near future. self.hash_algorithms.add('0x020000fe') # 0x020000ff is PSA_ALG_ANY_HASH self.mac_algorithms.add('0x03007fff') self.ka_algorithms.add('0x09fc0000') self.kdf_algorithms.add('0x080000ff') self.pake_algorithms.add('0x0a0000ff') # For AEAD algorithms, the only variability is over the tag length, # and this only applies to known algorithms, so don't test an # unknown algorithm. def get_names(self, type_word: str) -> Set[str]: """Return the set of known names of values of the given type.""" return { 'status': self.statuses, 'algorithm': self.algorithms, 'ecc_curve': self.ecc_curves, 'dh_group': self.dh_groups, 'key_type': self.key_types, 'key_usage': self.key_usage_flags, }[type_word] # Regex for interesting header lines. # Groups: 1=macro name, 2=type, 3=argument list (optional). _header_line_re = \ re.compile(r'#define +' + r'(PSA_((?:(?:DH|ECC|KEY)_)?[A-Z]+)_\w+)' + r'(?:\(([^\n()]*)\))?') # Regex of macro names to exclude. _excluded_name_re = re.compile(r'_(?:GET|IS|OF)_|_(?:BASE|FLAG|MASK)\Z') # Additional excluded macros. _excluded_names = set([ # Macros that provide an alternative way to build the same # algorithm as another macro. 'PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG', 'PSA_ALG_FULL_LENGTH_MAC', # Auxiliary macro whose name doesn't fit the usual patterns for # auxiliary macros. 'PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG_CASE', ]) def parse_header_line(self, line: str) -> None: """Parse a C header line, looking for "#define PSA_xxx".""" m = re.match(self._header_line_re, line) if not m: return name = m.group(1) self.all_declared.add(name) if re.search(self._excluded_name_re, name) or \ name in self._excluded_names or \ self.is_internal_name(name): return dest = self.table_by_prefix.get(m.group(2)) if dest is None: return dest.add(name) if m.group(3): self.argspecs[name] = self._argument_split(m.group(3)) _nonascii_re = re.compile(rb'[^\x00-\x7f]+') #type: Pattern def parse_header(self, filename: str) -> None: """Parse a C header file, looking for "#define PSA_xxx".""" with read_file_lines(filename, binary=True) as lines: for line in lines: line = re.sub(self._nonascii_re, rb'', line).decode('ascii') self.parse_header_line(line) _macro_identifier_re = re.compile(r'[A-Z]\w+') def generate_undeclared_names(self, expr: str) -> Iterable[str]: for name in re.findall(self._macro_identifier_re, expr): if name not in self.all_declared: yield name def accept_test_case_line(self, function: str, argument: str) -> bool: #pylint: disable=unused-argument undeclared = list(self.generate_undeclared_names(argument)) if undeclared: raise Exception('Undeclared names in test case', undeclared) return True @staticmethod def normalize_argument(argument: str) -> str: """Normalize whitespace in the given C expression. The result uses the same whitespace as ` PSAMacroEnumerator.distribute_arguments`. """ return re.sub(r',', r', ', re.sub(r' +', r'', argument)) def add_test_case_line(self, function: str, argument: str) -> None: """Parse a test case data line, looking for algorithm metadata tests.""" sets = [] if function.endswith('_algorithm'): sets.append(self.algorithms) if function == 'key_agreement_algorithm' and \ argument.startswith('PSA_ALG_KEY_AGREEMENT('): # We only want *raw* key agreement algorithms as such, so # exclude ones that are already chained with a KDF. # Keep the expression as one to test as an algorithm. function = 'other_algorithm' sets += self.table_by_test_function[function] if self.accept_test_case_line(function, argument): for s in sets: s.add(self.normalize_argument(argument)) # Regex matching a *.data line containing a test function call and # its arguments. The actual definition is partly positional, but this # regex is good enough in practice. _test_case_line_re = re.compile(r'(?!depends_on:)(\w+):([^\n :][^:\n]*)') def parse_test_cases(self, filename: str) -> None: """Parse a test case file (*.data), looking for algorithm metadata tests.""" with read_file_lines(filename) as lines: for line in lines: m = re.match(self._test_case_line_re, line) if m: self.add_test_case_line(m.group(1), m.group(2))