#!/usr/bin/env python3 # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later """ This script checks the current state of the source code for minor issues, including incorrect file permissions, presence of tabs, non-Unix line endings, trailing whitespace, and presence of UTF-8 BOM. Note: requires python 3, must be run from Mbed TLS root. """ import argparse import codecs import inspect import logging import os import re import subprocess import sys try: from typing import FrozenSet, Optional, Pattern # pylint: disable=unused-import except ImportError: pass import scripts_path # pylint: disable=unused-import from mbedtls_dev import build_tree class FileIssueTracker: """Base class for file-wide issue tracking. To implement a checker that processes a file as a whole, inherit from this class and implement `check_file_for_issue` and define ``heading``. ``suffix_exemptions``: files whose name ends with a string in this set will not be checked. ``path_exemptions``: files whose path (relative to the root of the source tree) matches this regular expression will not be checked. This can be ``None`` to match no path. Paths are normalized and converted to ``/`` separators before matching. ``heading``: human-readable description of the issue """ suffix_exemptions = frozenset() #type: FrozenSet[str] path_exemptions = None #type: Optional[Pattern[str]] # heading must be defined in derived classes. # pylint: disable=no-member def __init__(self): self.files_with_issues = {} @staticmethod def normalize_path(filepath): """Normalize ``filepath`` with / as the directory separator.""" filepath = os.path.normpath(filepath) # On Windows, we may have backslashes to separate directories. # We need slashes to match exemption lists. seps = os.path.sep if os.path.altsep is not None: seps += os.path.altsep return '/'.join(filepath.split(seps)) def should_check_file(self, filepath): """Whether the given file name should be checked. Files whose name ends with a string listed in ``self.suffix_exemptions`` or whose path matches ``self.path_exemptions`` will not be checked. """ for files_exemption in self.suffix_exemptions: if filepath.endswith(files_exemption): return False if self.path_exemptions and \ re.match(self.path_exemptions, self.normalize_path(filepath)): return False return True def check_file_for_issue(self, filepath): """Check the specified file for the issue that this class is for. Subclasses must implement this method. """ raise NotImplementedError def record_issue(self, filepath, line_number): """Record that an issue was found at the specified location.""" if filepath not in self.files_with_issues.keys(): self.files_with_issues[filepath] = [] self.files_with_issues[filepath].append(line_number) def output_file_issues(self, logger): """Log all the locations where the issue was found.""" if self.files_with_issues.values(): logger.info(self.heading) for filename, lines in sorted(self.files_with_issues.items()): if lines: logger.info("{}: {}".format( filename, ", ".join(str(x) for x in lines) )) else: logger.info(filename) logger.info("") BINARY_FILE_PATH_RE_LIST = [ r'docs/.*\.pdf\Z', r'docs/.*\.png\Z', r'programs/fuzz/corpuses/[^.]+\Z', r'tests/data_files/[^.]+\Z', r'tests/data_files/.*\.(crt|csr|db|der|key|pubkey)\Z', r'tests/data_files/.*\.req\.[^/]+\Z', r'tests/data_files/.*malformed[^/]+\Z', r'tests/data_files/format_pkcs12\.fmt\Z', r'tests/data_files/.*\.bin\Z', ] BINARY_FILE_PATH_RE = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST)) class LineIssueTracker(FileIssueTracker): """Base class for line-by-line issue tracking. To implement a checker that processes files line by line, inherit from this class and implement `line_with_issue`. """ # Exclude binary files. path_exemptions = BINARY_FILE_PATH_RE def issue_with_line(self, line, filepath, line_number): """Check the specified line for the issue that this class is for. Subclasses must implement this method. """ raise NotImplementedError def check_file_line(self, filepath, line, line_number): if self.issue_with_line(line, filepath, line_number): self.record_issue(filepath, line_number) def check_file_for_issue(self, filepath): """Check the lines of the specified file. Subclasses must implement the ``issue_with_line`` method. """ with open(filepath, "rb") as f: for i, line in enumerate(iter(f.readline, b"")): self.check_file_line(filepath, line, i + 1) def is_windows_file(filepath): _root, ext = os.path.splitext(filepath) return ext in ('.bat', '.dsp', '.dsw', '.sln', '.vcxproj') class ShebangIssueTracker(FileIssueTracker): """Track files with a bad, missing or extraneous shebang line. Executable scripts must start with a valid shebang (#!) line. """ heading = "Invalid shebang line:" # Allow either /bin/sh, /bin/bash, or /usr/bin/env. # Allow at most one argument (this is a Linux limitation). # For sh and bash, the argument if present must be options. # For env, the argument must be the base name of the interpreter. _shebang_re = re.compile(rb'^#! ?(?:/bin/(bash|sh)(?: -[^\n ]*)?' rb'|/usr/bin/env ([^\n /]+))$') _extensions = { b'bash': 'sh', b'perl': 'pl', b'python3': 'py', b'sh': 'sh', } path_exemptions = re.compile(r'tests/scripts/quiet/.*') def is_valid_shebang(self, first_line, filepath): m = re.match(self._shebang_re, first_line) if not m: return False interpreter = m.group(1) or m.group(2) if interpreter not in self._extensions: return False if not filepath.endswith('.' + self._extensions[interpreter]): return False return True def check_file_for_issue(self, filepath): is_executable = os.access(filepath, os.X_OK) with open(filepath, "rb") as f: first_line = f.readline() if first_line.startswith(b'#!'): if not is_executable: # Shebang on a non-executable file self.files_with_issues[filepath] = None elif not self.is_valid_shebang(first_line, filepath): self.files_with_issues[filepath] = [1] elif is_executable: # Executable without a shebang self.files_with_issues[filepath] = None class EndOfFileNewlineIssueTracker(FileIssueTracker): """Track files that end with an incomplete line (no newline character at the end of the last line).""" heading = "Missing newline at end of file:" path_exemptions = BINARY_FILE_PATH_RE def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: try: f.seek(-1, 2) except OSError: # This script only works on regular files. If we can't seek # 1 before the end, it means that this position is before # the beginning of the file, i.e. that the file is empty. return if f.read(1) != b"\n": self.files_with_issues[filepath] = None class Utf8BomIssueTracker(FileIssueTracker): """Track files that start with a UTF-8 BOM. Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM.""" heading = "UTF-8 BOM present:" suffix_exemptions = frozenset([".vcxproj", ".sln"]) path_exemptions = BINARY_FILE_PATH_RE def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: if f.read().startswith(codecs.BOM_UTF8): self.files_with_issues[filepath] = None class UnicodeIssueTracker(LineIssueTracker): """Track lines with invalid characters or invalid text encoding.""" heading = "Invalid UTF-8 or forbidden character:" # Only allow valid UTF-8, and only other explicitly allowed characters. # We deliberately exclude all characters that aren't a simple non-blank, # non-zero-width glyph, apart from a very small set (tab, ordinary space, # line breaks, "basic" no-break space and soft hyphen). In particular, # non-ASCII control characters, combinig characters, and Unicode state # changes (e.g. right-to-left text) are forbidden. # Note that we do allow some characters with a risk of visual confusion, # for example '-' (U+002D HYPHEN-MINUS) vs '­' (U+00AD SOFT HYPHEN) vs # '‐' (U+2010 HYPHEN), or 'A' (U+0041 LATIN CAPITAL LETTER A) vs # 'Α' (U+0391 GREEK CAPITAL LETTER ALPHA). GOOD_CHARACTERS = ''.join([ '\t\n\r -~', # ASCII (tabs and line endings are checked separately) '\u00A0-\u00FF', # Latin-1 Supplement (for NO-BREAK SPACE and punctuation) '\u2010-\u2027\u2030-\u205E', # General Punctuation (printable) '\u2070\u2071\u2074-\u208E\u2090-\u209C', # Superscripts and Subscripts '\u2190-\u21FF', # Arrows '\u2200-\u22FF', # Mathematical Symbols '\u2500-\u257F' # Box Drawings characters used in markdown trees ]) # Allow any of the characters and ranges above, and anything classified # as a word constituent. GOOD_CHARACTERS_RE = re.compile(r'[\w{}]+\Z'.format(GOOD_CHARACTERS)) def issue_with_line(self, line, _filepath, line_number): try: text = line.decode('utf-8') except UnicodeDecodeError: return True if line_number == 1 and text.startswith('\uFEFF'): # Strip BOM (U+FEFF ZERO WIDTH NO-BREAK SPACE) at the beginning. # Which files are allowed to have a BOM is handled in # Utf8BomIssueTracker. text = text[1:] return not self.GOOD_CHARACTERS_RE.match(text) class UnixLineEndingIssueTracker(LineIssueTracker): """Track files with non-Unix line endings (i.e. files with CR).""" heading = "Non-Unix line endings:" def should_check_file(self, filepath): if not super().should_check_file(filepath): return False return not is_windows_file(filepath) def issue_with_line(self, line, _filepath, _line_number): return b"\r" in line class WindowsLineEndingIssueTracker(LineIssueTracker): """Track files with non-Windows line endings (i.e. CR or LF not in CRLF).""" heading = "Non-Windows line endings:" def should_check_file(self, filepath): if not super().should_check_file(filepath): return False return is_windows_file(filepath) def issue_with_line(self, line, _filepath, _line_number): return not line.endswith(b"\r\n") or b"\r" in line[:-2] class TrailingWhitespaceIssueTracker(LineIssueTracker): """Track lines with trailing whitespace.""" heading = "Trailing whitespace:" suffix_exemptions = frozenset([".dsp", ".md"]) def issue_with_line(self, line, _filepath, _line_number): return line.rstrip(b"\r\n") != line.rstrip() class TabIssueTracker(LineIssueTracker): """Track lines with tabs.""" heading = "Tabs present:" suffix_exemptions = frozenset([ ".make", ".pem", # some openssl dumps have tabs ".sln", "/.gitmodules", "/Makefile", "/Makefile.inc", "/generate_visualc_files.pl", ]) def issue_with_line(self, line, _filepath, _line_number): return b"\t" in line class MergeArtifactIssueTracker(LineIssueTracker): """Track lines with merge artifacts. These are leftovers from a ``git merge`` that wasn't fully edited.""" heading = "Merge artifact:" def issue_with_line(self, line, _filepath, _line_number): # Detect leftover git conflict markers. if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '): return True if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3 return True if line.rstrip(b'\r\n') == b'=======' and \ not _filepath.endswith('.md'): return True return False def this_location(): frame = inspect.currentframe() assert frame is not None info = inspect.getframeinfo(frame) return os.path.basename(info.filename), info.lineno THIS_FILE_BASE_NAME, LINE_NUMBER_BEFORE_LICENSE_ISSUE_TRACKER = this_location() class LicenseIssueTracker(LineIssueTracker): """Check copyright statements and license indications. This class only checks that statements are correct if present. It does not enforce the presence of statements in each file. """ heading = "License issue:" LICENSE_EXEMPTION_RE_LIST = [ # Third-party code, other than whitelisted third-party modules, # may be under a different license. r'3rdparty/(?!(p256-m)/.*)', # Documentation explaining the license may have accidental # false positives. r'(ChangeLog|LICENSE|[-0-9A-Z_a-z]+\.md)\Z', # Files imported from TF-M, and not used except in test builds, # may be under a different license. r'configs/ext/crypto_config_profile_medium\.h\Z', r'configs/ext/tfm_mbedcrypto_config_profile_medium\.h\Z', r'configs/ext/README\.md\Z', # Third-party file. r'dco\.txt\Z', ] path_exemptions = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST + LICENSE_EXEMPTION_RE_LIST)) COPYRIGHT_HOLDER = rb'The Mbed TLS Contributors' # Catch "Copyright foo", "Copyright (C) foo", "Copyright © foo", etc. COPYRIGHT_RE = re.compile(rb'.*\bcopyright\s+((?:\w|\s|[()]|[^ -~])*\w)', re.I) SPDX_HEADER_KEY = b'SPDX-License-Identifier' LICENSE_IDENTIFIER = b'Apache-2.0 OR GPL-2.0-or-later' SPDX_RE = re.compile(br'.*?(' + re.escape(SPDX_HEADER_KEY) + br')(:\s*(.*?)\W*\Z|.*)', re.I) LICENSE_MENTION_RE = re.compile(rb'.*(?:' + rb'|'.join([ rb'Apache License', rb'General Public License', ]) + rb')', re.I) def __init__(self): super().__init__() # Record what problem was caused. We can't easily report it due to # the structure of the script. To be fixed after # https://github.com/Mbed-TLS/mbedtls/pull/2506 self.problem = None def issue_with_line(self, line, filepath, line_number): #pylint: disable=too-many-return-statements # Use endswith() rather than the more correct os.path.basename() # because experimentally, it makes a significant difference to # the running time. if filepath.endswith(THIS_FILE_BASE_NAME) and \ line_number > LINE_NUMBER_BEFORE_LICENSE_ISSUE_TRACKER: # Avoid false positives from the code in this class. # Also skip the rest of this file, which is highly unlikely to # contain any problematic statements since we put those near the # top of files. return False m = self.COPYRIGHT_RE.match(line) if m and m.group(1) != self.COPYRIGHT_HOLDER: self.problem = 'Invalid copyright line' return True m = self.SPDX_RE.match(line) if m: if m.group(1) != self.SPDX_HEADER_KEY: self.problem = 'Misspelled ' + self.SPDX_HEADER_KEY.decode() return True if not m.group(3): self.problem = 'Improperly formatted SPDX license identifier' return True if m.group(3) != self.LICENSE_IDENTIFIER: self.problem = 'Wrong SPDX license identifier' return True m = self.LICENSE_MENTION_RE.match(line) if m: self.problem = 'Suspicious license mention' return True return False class IntegrityChecker: """Sanity-check files under the current directory.""" def __init__(self, log_file): """Instantiate the sanity checker. Check files under the current directory. Write a report of issues to log_file.""" build_tree.check_repo_path() self.logger = None self.setup_logger(log_file) self.issues_to_check = [ ShebangIssueTracker(), EndOfFileNewlineIssueTracker(), Utf8BomIssueTracker(), UnicodeIssueTracker(), UnixLineEndingIssueTracker(), WindowsLineEndingIssueTracker(), TrailingWhitespaceIssueTracker(), TabIssueTracker(), MergeArtifactIssueTracker(), LicenseIssueTracker(), ] def setup_logger(self, log_file, level=logging.INFO): """Log to log_file if provided, or to stderr if None.""" self.logger = logging.getLogger() self.logger.setLevel(level) if log_file: handler = logging.FileHandler(log_file) self.logger.addHandler(handler) else: console = logging.StreamHandler() self.logger.addHandler(console) @staticmethod def collect_files(): """Return the list of files to check. These are the regular files commited into Git. """ bytes_output = subprocess.check_output(['git', 'ls-files', '-z']) bytes_filepaths = bytes_output.split(b'\0')[:-1] ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths) # Filter out directories. Normally Git doesn't list directories # (it only knows about the files inside them), but there is # at least one case where 'git ls-files' includes a directory: # submodules. Just skip submodules (and any other directories). ascii_filepaths = [fp for fp in ascii_filepaths if os.path.isfile(fp)] # Prepend './' to files in the top-level directory so that # something like `'/Makefile' in fp` matches in the top-level # directory as well as in subdirectories. return [fp if os.path.dirname(fp) else os.path.join(os.curdir, fp) for fp in ascii_filepaths] def check_files(self): """Check all files for all issues.""" for issue_to_check in self.issues_to_check: for filepath in self.collect_files(): if issue_to_check.should_check_file(filepath): issue_to_check.check_file_for_issue(filepath) def output_issues(self): """Log the issues found and their locations. Return 1 if there were issues, 0 otherwise. """ integrity_return_code = 0 for issue_to_check in self.issues_to_check: if issue_to_check.files_with_issues: integrity_return_code = 1 issue_to_check.output_file_issues(self.logger) return integrity_return_code def run_main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-l", "--log_file", type=str, help="path to optional output log", ) check_args = parser.parse_args() integrity_check = IntegrityChecker(check_args.log_file) integrity_check.check_files() return_code = integrity_check.output_issues() sys.exit(return_code) if __name__ == "__main__": run_main()