#!/usr/bin/env python3
# Copyright 2016 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script takes a Clang git revision as an argument, it then
creates a feature branch, puts this revision into update.py, uploads
a CL, triggers Clang Upload try bots, and tells what to do next"""
from __future__ import print_function
import argparse
import itertools
import os
import re
import subprocess
import sys
import urllib.request
from build import (CheckoutGitRepo, GetCommitDescription, GetLatestLLVMCommit,
LLVM_DIR, LLVM_GIT_URL, RunCommand)
from update import CHROMIUM_DIR, DownloadAndUnpack
# Access to //tools/rust
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..',
'rust'))
from build_rust import RUST_GIT_URL, RUST_SRC_DIR, GetLatestRustCommit
# Path constants.
THIS_DIR = os.path.dirname(__file__)
CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
CLANG_UPDATE_PY_PATH = os.path.join(THIS_DIR, 'update.py')
RUST_UPDATE_PY_PATH = os.path.join(THIS_DIR, '..', '..', 'rust',
'update_rust.py')
BUILD_RUST_PY_PATH = os.path.join(THIS_DIR, '..', '..', 'rust', 'build_rust.py')
# Bots where we build Clang + Rust.
BUILD_CLANG_BOTS = [
'linux_upload_clang',
'mac_upload_clang',
'mac_upload_clang_arm',
'win_upload_clang',
]
BUILD_RUST_BOTS = [
'linux_upload_rust',
'mac_upload_rust',
'mac_upload_rust_arm',
'win_upload_rust',
]
# Keep lines in here at <= 72 columns, else they wrap in gerrit.
# There can be no whitespace line between or below these gerrit footers.
COMMIT_FOOTER = \
'''
Bug: TODO. Remove the Tricium: line below when filling this in.
Tricium: skip
Disable-Rts: True
Cq-Include-Trybots: chromium/try:chromeos-amd64-generic-cfi-thin-lto-rel
Cq-Include-Trybots: chromium/try:dawn-win10-x86-deps-rel
Cq-Include-Trybots: chromium/try:lacros-arm64-generic-rel
Cq-Include-Trybots: chromium/try:linux-chromeos-dbg
Cq-Include-Trybots: chromium/try:linux_chromium_cfi_rel_ng
Cq-Include-Trybots: chromium/try:linux_chromium_chromeos_msan_rel_ng
Cq-Include-Trybots: chromium/try:linux_chromium_msan_rel_ng
Cq-Include-Trybots: chromium/try:mac11-arm64-rel,mac_chromium_asan_rel_ng
Cq-Include-Trybots: chromium/try:ios-catalyst,win-asan,android-official
Cq-Include-Trybots: chromium/try:fuchsia-arm64-cast-receiver-rel
Cq-Include-Trybots: chromium/try:mac-official,linux-official
Cq-Include-Trybots: chromium/try:win-official,win32-official
Cq-Include-Trybots: chromium/try:win-arm64-rel
Cq-Include-Trybots: chromium/try:linux-swangle-try-x64,win-swangle-try-x86
Cq-Include-Trybots: chromium/try:android-cronet-mainline-clang-arm64-dbg
Cq-Include-Trybots: chromium/try:android-cronet-mainline-clang-arm64-rel
Cq-Include-Trybots: chromium/try:android-cronet-mainline-clang-riscv64-dbg
Cq-Include-Trybots: chromium/try:android-cronet-mainline-clang-riscv64-rel
Cq-Include-Trybots: chromium/try:android-cronet-mainline-clang-x86-dbg
Cq-Include-Trybots: chromium/try:android-cronet-mainline-clang-x86-rel
Cq-Include-Trybots: chromium/try:android-cronet-riscv64-dbg
Cq-Include-Trybots: chromium/try:android-cronet-riscv64-rel
Cq-Include-Trybots: chrome/try:iphone-device,ipad-device
Cq-Include-Trybots: chrome/try:linux-chromeos-chrome
Cq-Include-Trybots: chrome/try:win-chrome,win64-chrome,linux-chrome,mac-chrome
Cq-Include-Trybots: chrome/try:linux-pgo,mac-pgo,win32-pgo,win64-pgo'''
RUST_BOTS = \
'''Cq-Include-Trybots: chromium/try:android-rust-arm32-rel
Cq-Include-Trybots: chromium/try:android-rust-arm64-dbg
Cq-Include-Trybots: chromium/try:android-rust-arm64-rel
Cq-Include-Trybots: chromium/try:linux-rust-x64-dbg
Cq-Include-Trybots: chromium/try:linux-rust-x64-rel
Cq-Include-Trybots: chromium/try:mac-rust-x64-dbg
Cq-Include-Trybots: chromium/try:win-rust-x64-dbg
Cq-Include-Trybots: chromium/try:win-rust-x64-rel'''
is_win = sys.platform.startswith('win32')
class RustVersion:
"""Holds the nightly Rust version in an explicit format."""
def __init__(self, git_hash: str, sub_revision: int):
self.git_hash = git_hash
self.short_git_hash = git_hash[slice(0, 12)]
self.sub_revision = sub_revision
def __str__(self) -> str:
"""A string containing the Rust version and sub revision.
The string is useful for humans, it contains all info needed to identify
the Rust version being built. It is also unique to a given Rust version and
subversion.
"""
return f'{self.git_hash}-{self.sub_revision}'
def __eq__(self, o) -> bool:
return (self.git_hash == o.git_hash and self.sub_revision == o.sub_revision)
class ClangVersion:
"""Holds the Clang version in an explicit format."""
def __init__(self, git_describe: str, sub_revision: str):
self.git_describe = git_describe
self.short_git_hash = re.search('-g([0-9a-f]+)', git_describe).group(1)
self.sub_revision = int(sub_revision)
def __str__(self) -> str:
"""A string containing the Clang version and sub revision.
The string is useful for humans, it contains all info needed to identify
the Clang version being built. It is also unique to a given Clang version
and subversion.
"""
return f'{self.git_describe}-{self.sub_revision}'
def __eq__(self, o) -> bool:
return (self.git_describe == o.git_describe
and self.sub_revision == o.sub_revision)
def PatchClangRevision(new_version: ClangVersion) -> ClangVersion:
with open(CLANG_UPDATE_PY_PATH) as f:
content = f.read()
REV = '\'([0-9a-z-]+)\''
SUB_REV = '([0-9]+)'
git_describe = re.search(f'CLANG_REVISION = {REV}', content).group(1)
sub_revision = re.search(f'CLANG_SUB_REVISION = {SUB_REV}', content).group(1)
old_version = ClangVersion(git_describe, sub_revision)
content = re.sub(f'CLANG_REVISION = {REV}',
f'CLANG_REVISION = \'{new_version.git_describe}\'',
content,
count=1)
content = re.sub(f'CLANG_SUB_REVISION = {SUB_REV}',
f'CLANG_SUB_REVISION = {new_version.sub_revision}',
content,
count=1)
with open(CLANG_UPDATE_PY_PATH, 'w') as f:
f.write(content)
return old_version
def PatchRustRevision(new_version: RustVersion) -> RustVersion:
with open(RUST_UPDATE_PY_PATH) as f:
content = f.read()
REV = '\'([0-9a-z-]+)\''
SUB_REV = '([0-9]+)'
git_hash = re.search(f'RUST_REVISION = {REV}', content).group(1)
sub_revision = re.search(f'RUST_SUB_REVISION = {SUB_REV}', content).group(1)
old_version = RustVersion(git_hash, sub_revision)
content = re.sub(f'RUST_REVISION = {REV}',
f'RUST_REVISION = \'{new_version.git_hash}\'',
content,
count=1)
content = re.sub(f'RUST_SUB_REVISION = {SUB_REV}',
f'RUST_SUB_REVISION = {new_version.sub_revision}',
content,
count=1)
with open(RUST_UPDATE_PY_PATH, 'w') as f:
f.write(content)
return old_version
def PatchRustStage0():
verify_stage0 = subprocess.run(
[sys.executable, BUILD_RUST_PY_PATH, '--verify-stage0-hash'],
capture_output=True,
text=True)
if verify_stage0.returncode == 0:
return
# TODO(crbug.com/40252478): We're printing a warning that the hash has
# changed, but we could require a verification step of some sort here. We
# should do the same for both Rust and Clang if we do so.
print(verify_stage0.stdout)
lines = verify_stage0.stdout.splitlines()
m = re.match('Actual hash: +([0-9a-z]+)', lines[-1])
new_stage0_hash = m.group(1)
with open(RUST_UPDATE_PY_PATH) as f:
content = f.read()
STAGE0_HASH = '\'([0-9a-z]+)\''
content = re.sub(f'STAGE0_JSON_SHA256 = {STAGE0_HASH}',
f'STAGE0_JSON_SHA256 = \'{new_stage0_hash}\'',
content,
count=1)
with open(RUST_UPDATE_PY_PATH, 'w') as f:
f.write(content)
def PatchRustRemoveOverride():
with open(RUST_UPDATE_PY_PATH) as f:
content = f.read()
REV = '([0-9a-z-]+)'
content = re.sub(f'OVERRIDE_CLANG_REVISION = \'{REV}\'',
f'OVERRIDE_CLANG_REVISION = None',
content,
count=1)
with open(RUST_UPDATE_PY_PATH, 'w') as f:
f.write(content)
def Git(*args, no_run: bool):
"""Runs a git command, or just prints it out if `no_run` is True."""
if no_run:
print('\033[91m', end='') # Color red
print('Skipped running: ', end='')
print('\033[0m', end='') # No color
print(*['git'] + [f'\'{i}\'' for i in list(args)], end='')
print()
else:
# Needs shell=True on Windows due to git.bat in depot_tools.
subprocess.check_call(['git'] + list(args), shell=is_win)
def main():
parser = argparse.ArgumentParser(description='upload new clang revision')
# TODO(crbug.com/40250560): Remove this when the cron job doesn't pass a SHA.
parser.add_argument(
'ignored',
nargs='?',
help='Ignored argument to handle the cron job passing a clang SHA')
parser.add_argument('--clang-git-hash',
type=str,
metavar='SHA1',
help='Clang git hash to build the toolchain for.')
parser.add_argument(
'--clang-sub-revision',
type=int,
default=1,
metavar='NUM',
help='Clang sub-revision to build the toolchain for. Defaults to 1.')
parser.add_argument('--rust-git-hash',
type=str,
metavar='SHA1',
help='Rust git hash to build the toolchain for.')
parser.add_argument(
'--rust-sub-revision',
type=int,
default=1,
metavar='NUM',
help='Rust sub-revision to build the toolchain for. Defaults to 1.')
parser.add_argument(
'--no-git',
action='store_true',
default=False,
help=('Print out `git` commands instead of running them. Still generates '
'a local diff for debugging purposes.'))
parser.add_argument('--skip-rust',
action='store_true',
default=False,
help=('Skip updating the rust revision.'))
parser.add_argument('--skip-clang',
action='store_true',
default=False,
help=('Skip updating the clang revision.'))
args = parser.parse_args()
if args.skip_clang and args.skip_rust:
print('Cannot set both --skip-clang and --skip-rust.')
sys.exit(1)
if args.skip_clang:
clang_version = '-skipped-'
else:
if args.clang_git_hash:
clang_git_hash = args.clang_git_hash
else:
clang_git_hash = GetLatestLLVMCommit()
# To `GetCommitDescription()`, we need a checkout. On success, the
# CheckoutLLVM() makes `LLVM_DIR` be the current working directory, so that
# we can GetCommitDescription() without changing directory.
CheckoutGitRepo("LLVM", LLVM_GIT_URL, clang_git_hash, LLVM_DIR)
clang_version = ClangVersion(GetCommitDescription(clang_git_hash),
args.clang_sub_revision)
os.chdir(CHROMIUM_DIR)
if args.skip_rust:
rust_version = '-skipped-'
else:
if args.rust_git_hash:
rust_git_hash = args.rust_git_hash
else:
rust_git_hash = GetLatestRustCommit()
CheckoutGitRepo("Rust", RUST_GIT_URL, rust_git_hash, RUST_SRC_DIR)
rust_version = RustVersion(rust_git_hash, args.rust_sub_revision)
os.chdir(CHROMIUM_DIR)
print(f'Making a patch for Clang {clang_version} and Rust {rust_version}')
branch_name = f'clang-{clang_version}_rust-{rust_version}'
Git('checkout', 'origin/main', '-b', branch_name, no_run=args.no_git)
old_clang_version = clang_version
if not args.skip_clang:
old_clang_version = PatchClangRevision(clang_version)
if args.skip_rust:
assert (clang_version !=
old_clang_version), ('Change the sub-revision of Clang if there is '
'no major version change.')
else:
old_rust_version = PatchRustRevision(rust_version)
assert (clang_version != old_clang_version
or rust_version != old_rust_version), (
'Change the sub-revision of Clang or Rust if there is '
'no major version change.')
PatchRustStage0()
if not args.skip_clang:
PatchRustRemoveOverride()
if args.skip_clang:
clang_change = '[skipping Clang]'
clang_change_log = ''
else:
clang_change = f'{old_clang_version} : {clang_version}'
clang_change_log = (
f'{LLVM_GIT_URL}/+log/'
f'{old_clang_version.short_git_hash}..{clang_version.short_git_hash}'
f'\n\n')
if args.skip_rust:
rust_change = '[skipping Rust]'
rust_change_log = ''
else:
rust_change = f'{old_rust_version} : {rust_version}'
rust_change_log = (f'{RUST_GIT_URL}/+log/'
f'{old_rust_version.short_git_hash}..'
f'{rust_version.short_git_hash}'
f'\n\n')
title = f'Roll clang+rust {clang_change} / {rust_change}'
cmd = ' '.join(sys.argv)
body = f'{clang_change_log}{rust_change_log}Ran: {cmd}'
commit_message = f'{title}\n\n{body}\n{COMMIT_FOOTER}'
if not args.skip_rust:
commit_message += f'\n{RUST_BOTS}'
Git('add',
CLANG_UPDATE_PY_PATH,
RUST_UPDATE_PY_PATH,
no_run=args.no_git)
Git('commit', '-m', commit_message, no_run=args.no_git)
Git('cl', 'upload', '-f', '--bypass-hooks', '--squash', no_run=args.no_git)
if not args.skip_clang:
Git('cl',
'try',
'-B',
"chromium/try",
*itertools.chain(*[['-b', bot] for bot in BUILD_CLANG_BOTS]),
no_run=args.no_git)
Git('cl',
'try',
'-B',
"chromium/try",
*itertools.chain(*[['-b', bot] for bot in BUILD_RUST_BOTS]),
no_run=args.no_git)
print('Please, wait until the try bots succeeded '
'and then push the binaries to RBE.')
print()
print('To update the Clang/Rust DEPS entries, run:\n '
'tools/clang/scripts/sync_deps.py')
print()
print('To regenerate BUILD.gn rules for Rust stdlib (needed if dep versions '
'in the stdlib change for example), run:\n tools/rust/gnrt_stdlib.py.')
print()
print('To update Abseil .def files, run:\n '
'third_party/abseil-cpp/generate_def_files.py')
if __name__ == '__main__':
sys.exit(main())
Codebase Browser
chromium