# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import logging
import os
import pathlib
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.request
import scripthash
_THIS_DIR = pathlib.Path(__file__).resolve().parent
_SRC_ROOT = _THIS_DIR.parents[1]
_CHECKOUT_SRC_ROOT_SUBDIR = '.3pp/chromium'
def parse_args():
parser = argparse.ArgumentParser()
# TODO(agrieve): Add required=True once 3pp builds with > python3.6.
subparsers = parser.add_subparsers()
subparser = subparsers.add_parser(
'latest', help='Prints the version as $LATEST.$RUNTIME_DEPS_HASH')
subparser.set_defaults(action='latest')
subparser = subparsers.add_parser(
'checkout', help='Copies files into the workdir used by docker')
subparser.add_argument('checkout_dir')
subparser.add_argument('--version', help='Output from "latest"')
subparser.set_defaults(action='checkout')
subparser = subparsers.add_parser(
'install',
help=('Run from workdir, inside docker container. '
'Builds & copies outputs into |output_prefix| directory'))
subparser.add_argument('output_prefix',
help='The path to install the compiled package to.')
subparser.add_argument('deps_prefix',
help='The path to a directory containing all deps.')
subparser.add_argument('--version', help='Output from "latest"')
subparser.add_argument('--checkout-dir', help='Directory to use as CWD')
subparser.set_defaults(action='install')
subparser = subparsers.add_parser(
'local-test', help='Run latest / checkout / install locally')
subparser.add_argument('--checkout-dir',
default='3pp_workdir',
help='Workdir to use')
subparser.add_argument('--output-prefix',
default='3pp_out',
help='Directory for final artifacts')
subparser.set_defaults(action='local-test')
args = parser.parse_args()
if not hasattr(args, 'action'):
parser.print_help()
sys.exit(1)
if hasattr(args, 'version'):
args.version = args.version or os.environ.get('_3PP_VERSION')
if not args.version:
parser.error('Must set --version or _3PP_VERSION')
if hasattr(args, 'output_prefix') and args.output_prefix:
args.output_prefix = os.path.abspath(args.output_prefix)
if hasattr(args, 'checkout_dir') and args.checkout_dir:
args.checkout_dir = os.path.abspath(args.checkout_dir)
if args.action == 'checkout':
# 3pp bot recipe does this, so needed only when running locally.
os.makedirs(args.checkout_dir, exist_ok=True)
if args.action == 'install':
if args.checkout_dir:
logging.info('Setting CWD=%s', args.checkout_dir)
os.chdir(args.checkout_dir)
if not os.path.exists(_CHECKOUT_SRC_ROOT_SUBDIR):
parser.error(f'Does not exist: {_CHECKOUT_SRC_ROOT_SUBDIR}.'
f' Use --checkout-dir?')
# 3pp bot recipe does this, so needed only when running locally.
os.makedirs(args.output_prefix, exist_ok=True)
return args
def path_within_checkout(subpath):
return os.path.abspath(os.path.join(_CHECKOUT_SRC_ROOT_SUBDIR, subpath))
def _all_files(path):
if os.path.isfile(path):
return [path]
assert os.path.isdir(path), 'Not a file or dir: ' + path
all_paths = pathlib.Path(path).glob('**/*')
return [str(f) for f in all_paths if f.is_file()]
def _resolve_runtime_deps(runtime_deps):
ret = []
for p in runtime_deps:
if p.startswith('//'):
ret.append(os.path.relpath(str(_SRC_ROOT / p[2:])))
elif os.path.isabs(p):
ret.append(os.path.relpath(p))
else:
ret.append(p)
return ret
def copy_runtime_deps(checkout_dir, runtime_deps):
# Make 3pp_common scripts available in the docker container install.py
# will run in.
dest_dir = os.path.join(checkout_dir, _CHECKOUT_SRC_ROOT_SUBDIR)
for src_path in _resolve_runtime_deps(runtime_deps):
relpath = os.path.relpath(src_path, _SRC_ROOT)
dest_path = os.path.join(dest_dir, relpath)
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
if os.path.isfile(src_path):
shutil.copy(src_path, dest_path)
else:
shutil.copytree(src_path,
dest_path,
ignore=shutil.ignore_patterns('.*', '__pycache__'))
logging.info('Runtime deps:')
sys.stderr.write('\n'.join(_all_files(checkout_dir)) + '\n')
def download_file(url, dest):
logging.info('Downloading %s', url)
with urllib.request.urlopen(url) as r:
with open(dest, 'wb') as f:
shutil.copyfileobj(r, f)
def extract_tar(path, dest):
logging.info('Extracting %s to %s', path, dest)
with tarfile.open(path) as f:
f.extractall(dest)
def run_cmd(cmd, check=True, *args, **kwargs):
logging.info('Running: %s', shlex.join(cmd))
return subprocess.run(cmd, check=check, *args, **kwargs)
def apply_patches(patches_dir, checkout_dir):
for path in sorted(pathlib.Path(patches_dir).glob('*.patch')):
cmd = ['git', 'apply', '-v', str(path)]
run_cmd(cmd, cwd=checkout_dir)
def main(*, do_latest, do_install, runtime_deps):
logging.basicConfig(
level=logging.DEBUG,
format='%(levelname).1s %(relativeCreated)6d %(message)s')
args = parse_args()
runtime_deps = [str(_THIS_DIR)] + runtime_deps
if args.action == 'local-test':
logging.warning('Will use work dir: %s', args.checkout_dir)
logging.warning('Will use output dir: %s', args.output_prefix)
if os.path.exists(args.checkout_dir) and os.listdir(args.checkout_dir):
logging.warning(
'*** Work dir not empty. This often causes failures. ***')
time.sleep(4)
# Approximates what 3pp recipe does for minimal configs.
# https://source.chromium.org/search?q=symbol:Chromium3ppApi.execute&ss=chromium
prog = os.path.abspath(sys.argv[0])
cmd = [prog, 'latest']
version = run_cmd(cmd, stdout=subprocess.PIPE, text=True).stdout
os.environ['_3PP_VERSION'] = version
checkout_dir = args.checkout_dir
run_cmd([prog, 'checkout', checkout_dir])
run_cmd([prog, 'install', args.output_prefix, 'UNUSED-DEPS-DIR'],
cwd=checkout_dir)
logging.warning('Local test complete.')
return
if args.action == 'latest':
version = do_latest()
assert version, 'do_latest() returned ' + repr(version)
extra_paths = []
for p in _resolve_runtime_deps(runtime_deps):
extra_paths += _all_files(p)
deps_hash = scripthash.compute(extra_paths=extra_paths)
print(f'{version}.{deps_hash}')
return
# Remove the hash at the end: 30.4.0-alpha05.HASH => 30.4.0-alpha05
args.version = args.version.rsplit('.', 1)[0]
if args.action == 'checkout':
copy_runtime_deps(args.checkout_dir, runtime_deps)
return
assert args.action == 'install'
do_install(args)
prefix_len = len(args.output_prefix) + 1
logging.info(
'Contents of %s: \n%s\n', args.output_prefix,
'\n'.join(p[prefix_len:] for p in _all_files(args.output_prefix)))
Codebase Browser
chromium