#!/usr/bin/env python3
"""
This script:
- Builds clang with user-defined flags
- Uses that clang to build an instrumented clang, which can be used to collect
PGO samples
- Builds a user-defined set of sources (default: clang) to act as a
"benchmark" to generate a PGO profile
- Builds clang once more with the PGO profile generated above
This is a total of four clean builds of clang (by default). This may take a
while. :)
This scripts duplicates https://llvm.org/docs/AdvancedBuilds.html#multi-stage-pgo
Eventually, it will be updated to instead call the cmake cache mentioned there.
"""
import argparse
import collections
import multiprocessing
import os
import shlex
import shutil
import subprocess
import sys
### User configuration
# If you want to use a different 'benchmark' than building clang, make this
# function do what you want. out_dir is the build directory for clang, so all
# of the clang binaries will live under "${out_dir}/bin/". Using clang in
# ${out_dir} will magically have the profiles go to the right place.
#
# You may assume that out_dir is a freshly-built directory that you can reach
# in to build more things, if you'd like.
def _run_benchmark(env, out_dir, include_debug_info):
"""The 'benchmark' we run to generate profile data."""
target_dir = env.output_subdir("instrumentation_run")
# `check-llvm` and `check-clang` are cheap ways to increase coverage. The
# former lets us touch on the non-x86 backends a bit if configured, and the
# latter gives us more C to chew on (and will send us through diagnostic
# paths a fair amount, though the `if (stuff_is_broken) { diag() ... }`
# branches should still heavily be weighted in the not-taken direction,
# since we built all of LLVM/etc).
_build_things_in(env, out_dir, what=["check-llvm", "check-clang"])
# Building tblgen gets us coverage; don't skip it. (out_dir may also not
# have them anyway, but that's less of an issue)
cmake = _get_cmake_invocation_for_bootstrap_from(env, out_dir, skip_tablegens=False)
if include_debug_info:
cmake.add_flag("CMAKE_BUILD_TYPE", "RelWithDebInfo")
_run_fresh_cmake(env, cmake, target_dir)
# Just build all the things. The more data we have, the better.
_build_things_in(env, target_dir, what=["all"])
### Script
class CmakeInvocation:
_cflags = ["CMAKE_C_FLAGS", "CMAKE_CXX_FLAGS"]
_ldflags = [
"CMAKE_EXE_LINKER_FLAGS",
"CMAKE_MODULE_LINKER_FLAGS",
"CMAKE_SHARED_LINKER_FLAGS",
]
def __init__(self, cmake, maker, cmake_dir):
self._prefix = [cmake, "-G", maker, cmake_dir]
# Map of str -> (list|str).
self._flags = {}
for flag in CmakeInvocation._cflags + CmakeInvocation._ldflags:
self._flags[flag] = []
def add_new_flag(self, key, value):
self.add_flag(key, value, allow_overwrites=False)
def add_flag(self, key, value, allow_overwrites=True):
if key not in self._flags:
self._flags[key] = value
return
existing_value = self._flags[key]
if isinstance(existing_value, list):
existing_value.append(value)
return
if not allow_overwrites:
raise ValueError("Invalid overwrite of %s requested" % key)
self._flags[key] = value
def add_cflags(self, flags):
# No, I didn't intend to append ['-', 'O', '2'] to my flags, thanks :)
assert not isinstance(flags, str)
for f in CmakeInvocation._cflags:
self._flags[f].extend(flags)
def add_ldflags(self, flags):
assert not isinstance(flags, str)
for f in CmakeInvocation._ldflags:
self._flags[f].extend(flags)
def to_args(self):
args = self._prefix.copy()
for key, value in sorted(self._flags.items()):
if isinstance(value, list):
# We preload all of the list-y values (cflags, ...). If we've
# nothing to add, don't.
if not value:
continue
value = " ".join(value)
arg = "-D" + key
if value != "":
arg += "=" + value
args.append(arg)
return args
class Env:
def __init__(self, llvm_dir, use_make, output_dir, default_cmake_args, dry_run):
self.llvm_dir = llvm_dir
self.use_make = use_make
self.output_dir = output_dir
self.default_cmake_args = default_cmake_args.copy()
self.dry_run = dry_run
def get_default_cmake_args_kv(self):
return self.default_cmake_args.items()
def get_cmake_maker(self):
return "Ninja" if not self.use_make else "Unix Makefiles"
def get_make_command(self):
if self.use_make:
return ["make", "-j{}".format(multiprocessing.cpu_count())]
return ["ninja"]
def output_subdir(self, name):
return os.path.join(self.output_dir, name)
def has_llvm_subproject(self, name):
if name == "compiler-rt":
subdir = "../compiler-rt"
elif name == "clang":
subdir = "../clang"
else:
raise ValueError("Unknown subproject: %s" % name)
return os.path.isdir(os.path.join(self.llvm_dir, subdir))
# Note that we don't allow capturing stdout/stderr. This works quite nicely
# with dry_run.
def run_command(self, cmd, cwd=None, check=False, silent_unless_error=False):
print("Running `%s` in %s" % (cmd, shlex.quote(cwd or os.getcwd())))
if self.dry_run:
return
if silent_unless_error:
stdout, stderr = subprocess.PIPE, subprocess.STDOUT
else:
stdout, stderr = None, None
# Don't use subprocess.run because it's >= py3.5 only, and it's not too
# much extra effort to get what it gives us anyway.
popen = subprocess.Popen(
cmd, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, cwd=cwd
)
stdout, _ = popen.communicate()
return_code = popen.wait(timeout=0)
if not return_code:
return
if silent_unless_error:
print(stdout.decode("utf-8", "ignore"))
if check:
raise subprocess.CalledProcessError(
returncode=return_code, cmd=cmd, output=stdout, stderr=None
)
def _get_default_cmake_invocation(env):
inv = CmakeInvocation(
cmake="cmake", maker=env.get_cmake_maker(), cmake_dir=env.llvm_dir
)
for key, value in env.get_default_cmake_args_kv():
inv.add_new_flag(key, value)
return inv
def _get_cmake_invocation_for_bootstrap_from(env, out_dir, skip_tablegens=True):
clang = os.path.join(out_dir, "bin", "clang")
cmake = _get_default_cmake_invocation(env)
cmake.add_new_flag("CMAKE_C_COMPILER", clang)
cmake.add_new_flag("CMAKE_CXX_COMPILER", clang + "++")
# We often get no value out of building new tblgens; the previous build
# should have them. It's still correct to build them, just slower.
def add_tablegen(key, binary):
path = os.path.join(out_dir, "bin", binary)
# Check that this exists, since the user's allowed to specify their own
# stage1 directory (which is generally where we'll source everything
# from). Dry runs should hope for the best from our user, as well.
if env.dry_run or os.path.exists(path):
cmake.add_new_flag(key, path)
if skip_tablegens:
add_tablegen("LLVM_TABLEGEN", "llvm-tblgen")
add_tablegen("CLANG_TABLEGEN", "clang-tblgen")
return cmake
def _build_things_in(env, target_dir, what):
cmd = env.get_make_command() + what
env.run_command(cmd, cwd=target_dir, check=True)
def _run_fresh_cmake(env, cmake, target_dir):
if not env.dry_run:
try:
shutil.rmtree(target_dir)
except FileNotFoundError:
pass
os.makedirs(target_dir, mode=0o755)
cmake_args = cmake.to_args()
env.run_command(cmake_args, cwd=target_dir, check=True, silent_unless_error=True)
def _build_stage1_clang(env):
target_dir = env.output_subdir("stage1")
cmake = _get_default_cmake_invocation(env)
_run_fresh_cmake(env, cmake, target_dir)
_build_things_in(env, target_dir, what=["clang", "llvm-profdata", "profile"])
return target_dir
def _generate_instrumented_clang_profile(env, stage1_dir, profile_dir, output_file):
llvm_profdata = os.path.join(stage1_dir, "bin", "llvm-profdata")
if env.dry_run:
profiles = [os.path.join(profile_dir, "*.profraw")]
else:
profiles = [
os.path.join(profile_dir, f)
for f in os.listdir(profile_dir)
if f.endswith(".profraw")
]
cmd = [llvm_profdata, "merge", "-output=" + output_file] + profiles
env.run_command(cmd, check=True)
def _build_instrumented_clang(env, stage1_dir):
assert os.path.isabs(stage1_dir)
target_dir = os.path.join(env.output_dir, "instrumented")
cmake = _get_cmake_invocation_for_bootstrap_from(env, stage1_dir)
cmake.add_new_flag("LLVM_BUILD_INSTRUMENTED", "IR")
# libcxx's configure step messes with our link order: we'll link
# libclang_rt.profile after libgcc, and the former requires atexit from the
# latter. So, configure checks fail.
#
# Since we don't need libcxx or compiler-rt anyway, just disable them.
cmake.add_new_flag("LLVM_BUILD_RUNTIME", "No")
_run_fresh_cmake(env, cmake, target_dir)
_build_things_in(env, target_dir, what=["clang", "lld"])
profiles_dir = os.path.join(target_dir, "profiles")
return target_dir, profiles_dir
def _build_optimized_clang(env, stage1_dir, profdata_file):
if not env.dry_run and not os.path.exists(profdata_file):
raise ValueError(
"Looks like the profdata file at %s doesn't exist" % profdata_file
)
target_dir = os.path.join(env.output_dir, "optimized")
cmake = _get_cmake_invocation_for_bootstrap_from(env, stage1_dir)
cmake.add_new_flag("LLVM_PROFDATA_FILE", os.path.abspath(profdata_file))
# We'll get complaints about hash mismatches in `main` in tools/etc. Ignore
# it.
cmake.add_cflags(["-Wno-backend-plugin"])
_run_fresh_cmake(env, cmake, target_dir)
_build_things_in(env, target_dir, what=["clang"])
return target_dir
Args = collections.namedtuple(
"Args",
[
"do_optimized_build",
"include_debug_info",
"profile_location",
"stage1_dir",
],
)
def _parse_args():
parser = argparse.ArgumentParser(
description="Builds LLVM and Clang with instrumentation, collects "
"instrumentation profiles for them, and (optionally) builds things "
"with these PGO profiles. By default, it's assumed that you're "
"running this from your LLVM root, and all build artifacts will be "
"saved to $PWD/out."
)
parser.add_argument(
"--cmake-extra-arg",
action="append",
default=[],
help="an extra arg to pass to all cmake invocations. Note that this "
"is interpreted as a -D argument, e.g. --cmake-extra-arg FOO=BAR will "
"be passed as -DFOO=BAR. This may be specified multiple times.",
)
parser.add_argument(
"--dry-run", action="store_true", help="print commands instead of running them"
)
parser.add_argument(
"--llvm-dir",
default=".",
help="directory containing an LLVM checkout (default: $PWD)",
)
parser.add_argument(
"--no-optimized-build",
action="store_true",
help="disable the final, PGO-optimized build",
)
parser.add_argument(
"--out-dir", help="directory to write artifacts to (default: $llvm_dir/out)"
)
parser.add_argument(
"--profile-output",
help="where to output the profile (default is $out/pgo_profile.prof)",
)
parser.add_argument(
"--stage1-dir",
help="instead of having an initial build of everything, use the given "
"directory. It is expected that this directory will have clang, "
"llvm-profdata, and the appropriate libclang_rt.profile already built",
)
parser.add_argument(
"--use-debug-info-in-benchmark",
action="store_true",
help="use a regular build instead of RelWithDebInfo in the benchmark. "
"This increases benchmark execution time and disk space requirements, "
"but gives more coverage over debuginfo bits in LLVM and clang.",
)
parser.add_argument(
"--use-make",
action="store_true",
default=shutil.which("ninja") is None,
help="use Makefiles instead of ninja",
)
args = parser.parse_args()
llvm_dir = os.path.abspath(args.llvm_dir)
if args.out_dir is None:
output_dir = os.path.join(llvm_dir, "out")
else:
output_dir = os.path.abspath(args.out_dir)
extra_args = {
"CMAKE_BUILD_TYPE": "Release",
"LLVM_ENABLE_PROJECTS": "clang;compiler-rt;lld",
}
for arg in args.cmake_extra_arg:
if arg.startswith("-D"):
arg = arg[2:]
elif arg.startswith("-"):
raise ValueError(
"Unknown not- -D arg encountered; you may need "
"to tweak the source..."
)
split = arg.split("=", 1)
if len(split) == 1:
key, val = split[0], ""
else:
key, val = split
extra_args[key] = val
env = Env(
default_cmake_args=extra_args,
dry_run=args.dry_run,
llvm_dir=llvm_dir,
output_dir=output_dir,
use_make=args.use_make,
)
if args.profile_output is not None:
profile_location = args.profile_output
else:
profile_location = os.path.join(env.output_dir, "pgo_profile.prof")
result_args = Args(
do_optimized_build=not args.no_optimized_build,
include_debug_info=args.use_debug_info_in_benchmark,
profile_location=profile_location,
stage1_dir=args.stage1_dir,
)
return env, result_args
def _looks_like_llvm_dir(directory):
"""Arbitrary set of heuristics to determine if `directory` is an llvm dir.
Errs on the side of false-positives."""
contents = set(os.listdir(directory))
expected_contents = [
"CODE_OWNERS.TXT",
"cmake",
"docs",
"include",
"utils",
]
if not all(c in contents for c in expected_contents):
return False
try:
include_listing = os.listdir(os.path.join(directory, "include"))
except NotADirectoryError:
return False
return "llvm" in include_listing
def _die(*args, **kwargs):
kwargs["file"] = sys.stderr
print(*args, **kwargs)
sys.exit(1)
def _main():
env, args = _parse_args()
if not _looks_like_llvm_dir(env.llvm_dir):
_die("Looks like %s isn't an LLVM directory; please see --help" % env.llvm_dir)
if not env.has_llvm_subproject("clang"):
_die("Need a clang checkout at tools/clang")
if not env.has_llvm_subproject("compiler-rt"):
_die("Need a compiler-rt checkout at projects/compiler-rt")
def status(*args):
print(*args, file=sys.stderr)
if args.stage1_dir is None:
status("*** Building stage1 clang...")
stage1_out = _build_stage1_clang(env)
else:
stage1_out = args.stage1_dir
status("*** Building instrumented clang...")
instrumented_out, profile_dir = _build_instrumented_clang(env, stage1_out)
status("*** Running profdata benchmarks...")
_run_benchmark(env, instrumented_out, args.include_debug_info)
status("*** Generating profile...")
_generate_instrumented_clang_profile(
env, stage1_out, profile_dir, args.profile_location
)
print("Final profile:", args.profile_location)
if args.do_optimized_build:
status("*** Building PGO-optimized binaries...")
optimized_out = _build_optimized_clang(env, stage1_out, args.profile_location)
print("Final build directory:", optimized_out)
if __name__ == "__main__":
_main()
Codebase Browser
llvm