# Copyright 2024 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse import logging import os import pathlib import shlex import shutil import subprocess import sys import tarfile import tempfile import time import urllib.request import scripthash _THIS_DIR = pathlib.Path(__file__).resolve().parent _SRC_ROOT = _THIS_DIR.parents[1] _CHECKOUT_SRC_ROOT_SUBDIR = '.3pp/chromium' def parse_args(): parser = argparse.ArgumentParser() # TODO(agrieve): Add required=True once 3pp builds with > python3.6. subparsers = parser.add_subparsers() subparser = subparsers.add_parser( 'latest', help='Prints the version as $LATEST.$RUNTIME_DEPS_HASH') subparser.set_defaults(action='latest') subparser = subparsers.add_parser( 'checkout', help='Copies files into the workdir used by docker') subparser.add_argument('checkout_dir') subparser.add_argument('--version', help='Output from "latest"') subparser.set_defaults(action='checkout') subparser = subparsers.add_parser( 'install', help=('Run from workdir, inside docker container. ' 'Builds & copies outputs into |output_prefix| directory')) subparser.add_argument('output_prefix', help='The path to install the compiled package to.') subparser.add_argument('deps_prefix', help='The path to a directory containing all deps.') subparser.add_argument('--version', help='Output from "latest"') subparser.add_argument('--checkout-dir', help='Directory to use as CWD') subparser.set_defaults(action='install') subparser = subparsers.add_parser( 'local-test', help='Run latest / checkout / install locally') subparser.add_argument('--checkout-dir', default='3pp_workdir', help='Workdir to use') subparser.add_argument('--output-prefix', default='3pp_out', help='Directory for final artifacts') subparser.set_defaults(action='local-test') args = parser.parse_args() if not hasattr(args, 'action'): parser.print_help() sys.exit(1) if hasattr(args, 'version'): args.version = args.version or os.environ.get('_3PP_VERSION') if not args.version: parser.error('Must set --version or _3PP_VERSION') if hasattr(args, 'output_prefix') and args.output_prefix: args.output_prefix = os.path.abspath(args.output_prefix) if hasattr(args, 'checkout_dir') and args.checkout_dir: args.checkout_dir = os.path.abspath(args.checkout_dir) if args.action == 'checkout': # 3pp bot recipe does this, so needed only when running locally. os.makedirs(args.checkout_dir, exist_ok=True) if args.action == 'install': if args.checkout_dir: logging.info('Setting CWD=%s', args.checkout_dir) os.chdir(args.checkout_dir) if not os.path.exists(_CHECKOUT_SRC_ROOT_SUBDIR): parser.error(f'Does not exist: {_CHECKOUT_SRC_ROOT_SUBDIR}.' f' Use --checkout-dir?') # 3pp bot recipe does this, so needed only when running locally. os.makedirs(args.output_prefix, exist_ok=True) return args def path_within_checkout(subpath): return os.path.abspath(os.path.join(_CHECKOUT_SRC_ROOT_SUBDIR, subpath)) def _all_files(path): if os.path.isfile(path): return [path] assert os.path.isdir(path), 'Not a file or dir: ' + path all_paths = pathlib.Path(path).glob('**/*') return [str(f) for f in all_paths if f.is_file()] def _resolve_runtime_deps(runtime_deps): ret = [] for p in runtime_deps: if p.startswith('//'): ret.append(os.path.relpath(str(_SRC_ROOT / p[2:]))) elif os.path.isabs(p): ret.append(os.path.relpath(p)) else: ret.append(p) return ret def copy_runtime_deps(checkout_dir, runtime_deps): # Make 3pp_common scripts available in the docker container install.py # will run in. dest_dir = os.path.join(checkout_dir, _CHECKOUT_SRC_ROOT_SUBDIR) for src_path in _resolve_runtime_deps(runtime_deps): relpath = os.path.relpath(src_path, _SRC_ROOT) dest_path = os.path.join(dest_dir, relpath) os.makedirs(os.path.dirname(dest_path), exist_ok=True) if os.path.isfile(src_path): shutil.copy(src_path, dest_path) else: shutil.copytree(src_path, dest_path, ignore=shutil.ignore_patterns('.*', '__pycache__')) logging.info('Runtime deps:') sys.stderr.write('\n'.join(_all_files(checkout_dir)) + '\n') def download_file(url, dest): logging.info('Downloading %s', url) with urllib.request.urlopen(url) as r: with open(dest, 'wb') as f: shutil.copyfileobj(r, f) def extract_tar(path, dest): logging.info('Extracting %s to %s', path, dest) with tarfile.open(path) as f: f.extractall(dest) def run_cmd(cmd, check=True, *args, **kwargs): logging.info('Running: %s', shlex.join(cmd)) return subprocess.run(cmd, check=check, *args, **kwargs) def apply_patches(patches_dir, checkout_dir): for path in sorted(pathlib.Path(patches_dir).glob('*.patch')): cmd = ['git', 'apply', '-v', str(path)] run_cmd(cmd, cwd=checkout_dir) def main(*, do_latest, do_install, runtime_deps): logging.basicConfig( level=logging.DEBUG, format='%(levelname).1s %(relativeCreated)6d %(message)s') args = parse_args() runtime_deps = [str(_THIS_DIR)] + runtime_deps if args.action == 'local-test': logging.warning('Will use work dir: %s', args.checkout_dir) logging.warning('Will use output dir: %s', args.output_prefix) if os.path.exists(args.checkout_dir) and os.listdir(args.checkout_dir): logging.warning( '*** Work dir not empty. This often causes failures. ***') time.sleep(4) # Approximates what 3pp recipe does for minimal configs. # https://source.chromium.org/search?q=symbol:Chromium3ppApi.execute&ss=chromium prog = os.path.abspath(sys.argv[0]) cmd = [prog, 'latest'] version = run_cmd(cmd, stdout=subprocess.PIPE, text=True).stdout os.environ['_3PP_VERSION'] = version checkout_dir = args.checkout_dir run_cmd([prog, 'checkout', checkout_dir]) run_cmd([prog, 'install', args.output_prefix, 'UNUSED-DEPS-DIR'], cwd=checkout_dir) logging.warning('Local test complete.') return if args.action == 'latest': version = do_latest() assert version, 'do_latest() returned ' + repr(version) extra_paths = [] for p in _resolve_runtime_deps(runtime_deps): extra_paths += _all_files(p) deps_hash = scripthash.compute(extra_paths=extra_paths) print(f'{version}.{deps_hash}') return # Remove the hash at the end: 30.4.0-alpha05.HASH => 30.4.0-alpha05 args.version = args.version.rsplit('.', 1)[0] if args.action == 'checkout': copy_runtime_deps(args.checkout_dir, runtime_deps) return assert args.action == 'install' do_install(args) prefix_len = len(args.output_prefix) + 1 logging.info( 'Contents of %s: \n%s\n', args.output_prefix, '\n'.join(p[prefix_len:] for p in _all_files(args.output_prefix)))