#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-unsafe
import glob
import json
import os
import pathlib
import shutil
import stat
import subprocess
import sys
import typing
from typing import Optional
from .dyndeps import create_dyn_dep_munger
from .envfuncs import add_path_entry, Env, path_search
from .fetcher import copy_if_different
from .runcmd import run_cmd
if typing.TYPE_CHECKING:
from .buildopts import BuildOptions
class BuilderBase(object):
def __init__(
self,
loader,
dep_manifests, # manifests of dependencies
build_opts: "BuildOptions",
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
env=None,
final_install_prefix=None,
) -> None:
self.env = Env()
if env:
self.env.update(env)
subdir = manifest.get("build", "subdir", ctx=ctx)
if subdir:
src_dir = os.path.join(src_dir, subdir)
self.patchfile = manifest.get("build", "patchfile", ctx=ctx)
self.patchfile_opts = manifest.get("build", "patchfile_opts", ctx=ctx) or ""
self.ctx = ctx
self.src_dir = src_dir
self.build_dir = build_dir or src_dir
self.inst_dir = inst_dir
self.build_opts = build_opts
self.manifest = manifest
self.final_install_prefix = final_install_prefix
self.loader = loader
self.dep_manifests = dep_manifests
self.install_dirs = [loader.get_project_install_dir(m) for m in dep_manifests]
def _get_cmd_prefix(self):
if self.build_opts.is_windows():
vcvarsall = self.build_opts.get_vcvars_path()
if vcvarsall is not None:
# Since it sets rather a large number of variables we mildly abuse
# the cmd quoting rules to assemble a command that calls the script
# to prep the environment and then triggers the actual command that
# we wanted to run.
# Due to changes in vscrsall.bat, it now reports an ERRORLEVEL of 1
# even when succeeding. This occurs when an extension is not present.
# To continue, we must ignore the ERRORLEVEL returned. We do this by
# wrapping the call in a batch file that always succeeds.
wrapper = os.path.join(self.build_dir, "succeed.bat")
with open(wrapper, "w") as f:
f.write("@echo off\n")
f.write(f'call "{vcvarsall}" amd64\n')
f.write("set ERRORLEVEL=0\n")
f.write("exit /b 0\n")
return [wrapper, "&&"]
return []
def _run_cmd(
self,
cmd,
cwd=None,
env=None,
use_cmd_prefix: bool = True,
allow_fail: bool = False,
) -> int:
if env:
e = self.env.copy()
e.update(env)
env = e
else:
env = self.env
if use_cmd_prefix:
cmd_prefix = self._get_cmd_prefix()
if cmd_prefix:
cmd = cmd_prefix + cmd
log_file = os.path.join(self.build_dir, "getdeps_build.log")
return run_cmd(
cmd=cmd,
env=env,
cwd=cwd or self.build_dir,
log_file=log_file,
allow_fail=allow_fail,
)
def _reconfigure(self, reconfigure: bool) -> bool:
if self.build_dir is not None:
if not os.path.isdir(self.build_dir):
os.makedirs(self.build_dir)
reconfigure = True
return reconfigure
def _apply_patchfile(self) -> None:
if self.patchfile is None:
return
patched_sentinel_file = pathlib.Path(self.src_dir + "/.getdeps_patched")
if patched_sentinel_file.exists():
return
old_wd = os.getcwd()
os.chdir(self.src_dir)
print(f"Patching {self.manifest.name} with {self.patchfile} in {self.src_dir}")
patchfile = os.path.join(
self.build_opts.fbcode_builder_dir, "patches", self.patchfile
)
patchcmd = ["git", "apply", "--ignore-space-change"]
if self.patchfile_opts:
patchcmd.append(self.patchfile_opts)
try:
subprocess.check_call(patchcmd + [patchfile])
except subprocess.CalledProcessError:
raise ValueError(f"Failed to apply patch to {self.manifest.name}")
os.chdir(old_wd)
patched_sentinel_file.touch()
def prepare(self, reconfigure: bool) -> None:
print("Preparing %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile()
self._prepare(reconfigure=reconfigure)
def debug(self, reconfigure: bool) -> None:
reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile()
self._prepare(reconfigure=reconfigure)
env = self._compute_env()
print("Starting a shell in %s, ^D to exit..." % self.build_dir)
# TODO: print the command to run the build
shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"]
self._run_cmd(shell, cwd=self.build_dir, env=env)
def build(self, reconfigure: bool) -> None:
print("Building %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile()
self._prepare(reconfigure=reconfigure)
self._build(reconfigure=reconfigure)
if self.build_opts.free_up_disk:
# don't clean --src-dir=. case as user may want to build again or run tests on the build
if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir(
self.build_dir
):
if os.path.islink(self.build_dir):
os.remove(self.build_dir)
else:
shutil.rmtree(self.build_dir)
elif self.build_opts.is_windows():
# On Windows, emit a wrapper script that can be used to run build artifacts
# directly from the build directory, without installing them. On Windows $PATH
# needs to be updated to include all of the directories containing the runtime
# library dependencies in order to run the binaries.
script_path = self.get_dev_run_script_path()
dep_munger = create_dyn_dep_munger(
self.build_opts, self._compute_env(), self.install_dirs
)
dep_dirs = self.get_dev_run_extra_path_dirs(dep_munger)
# pyre-fixme[16]: Optional type has no attribute `emit_dev_run_script`.
dep_munger.emit_dev_run_script(script_path, dep_dirs)
@property
def num_jobs(self) -> int:
# This is a hack, but we don't have a "defaults manifest" that we can
# customize per platform.
# TODO: Introduce some sort of defaults config that can select by
# platform, just like manifest contexts.
if sys.platform.startswith("freebsd"):
# clang on FreeBSD is quite memory-efficient.
default_job_weight = 512
else:
# 1.5 GiB is a lot to assume, but it's typical of Facebook-style C++.
# Some manifests are even heavier and should override.
default_job_weight = 1536
return self.build_opts.get_num_jobs(
int(
self.manifest.get(
"build", "job_weight_mib", default_job_weight, ctx=self.ctx
)
)
)
def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> None:
"""Execute any tests that we know how to run. If they fail,
raise an exception."""
pass
def _prepare(self, reconfigure) -> None:
"""Prepare the build. Useful when need to generate config,
but builder is not the primary build system.
e.g. cargo when called from cmake"""
pass
def _build(self, reconfigure) -> None:
"""Perform the build.
reconfigure will be set to true if the fetcher determined
that the sources have changed in such a way that the build
system needs to regenerate its rules."""
pass
def _compute_env(self):
# CMAKE_PREFIX_PATH is only respected when passed through the
# environment, so we construct an appropriate path to pass down
return self.build_opts.compute_env_for_install_dirs(
self.loader,
self.dep_manifests,
self.ctx,
env=self.env,
manifest=self.manifest,
)
def get_dev_run_script_path(self):
assert self.build_opts.is_windows()
return os.path.join(self.build_dir, "run.ps1")
def get_dev_run_extra_path_dirs(self, dep_munger=None):
assert self.build_opts.is_windows()
if dep_munger is None:
dep_munger = create_dyn_dep_munger(
self.build_opts, self._compute_env(), self.install_dirs
)
return dep_munger.compute_dependency_paths(self.build_dir)
class MakeBuilder(BuilderBase):
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
build_args,
install_args,
test_args,
) -> None:
super(MakeBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
self.build_args = build_args or []
self.install_args = install_args or []
self.test_args = test_args
@property
def _make_binary(self):
return self.manifest.get("build", "make_binary", "make", ctx=self.ctx)
def _get_prefix(self):
return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
def _build(self, reconfigure) -> None:
env = self._compute_env()
# Need to ensure that PREFIX is set prior to install because
# libbpf uses it when generating its pkg-config file.
# The lowercase prefix is used by some projects.
cmd = (
[self._make_binary, "-j%s" % self.num_jobs]
+ self.build_args
+ self._get_prefix()
)
self._run_cmd(cmd, env=env)
install_cmd = [self._make_binary] + self.install_args + self._get_prefix()
self._run_cmd(install_cmd, env=env)
# bz2's Makefile doesn't install its .so properly
if self.manifest and self.manifest.name == "bz2":
libdir = os.path.join(self.inst_dir, "lib")
srcpattern = os.path.join(self.src_dir, "lib*.so.*")
print(f"copying to {libdir} from {srcpattern}")
for file in glob.glob(srcpattern):
shutil.copy(file, libdir)
def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> None:
if not self.test_args:
return
env = self._compute_env()
cmd = [self._make_binary] + self.test_args + self._get_prefix()
self._run_cmd(cmd, env=env)
class CMakeBootStrapBuilder(MakeBuilder):
def _build(self, reconfigure) -> None:
self._run_cmd(
[
"./bootstrap",
"--prefix=" + self.inst_dir,
f"--parallel={self.num_jobs}",
]
)
super(CMakeBootStrapBuilder, self)._build(reconfigure)
class AutoconfBuilder(BuilderBase):
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
args,
conf_env_args,
) -> None:
super(AutoconfBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
self.args = args or []
self.conf_env_args = conf_env_args or {}
@property
def _make_binary(self):
return self.manifest.get("build", "make_binary", "make", ctx=self.ctx)
def _build(self, reconfigure) -> None:
configure_path = os.path.join(self.src_dir, "configure")
autogen_path = os.path.join(self.src_dir, "autogen.sh")
env = self._compute_env()
# Some configure scripts need additional env values passed derived from cmds
for k, cmd_args in self.conf_env_args.items():
out = (
subprocess.check_output(cmd_args, env=dict(env.items()))
.decode("utf-8")
.strip()
)
if out:
env.set(k, out)
if not os.path.exists(configure_path):
print("%s doesn't exist, so reconfiguring" % configure_path)
# This libtoolize call is a bit gross; the issue is that
# `autoreconf` as invoked by libsodium's `autogen.sh` doesn't
# seem to realize that it should invoke libtoolize and then
# error out when the configure script references a libtool
# related symbol.
self._run_cmd(["libtoolize"], cwd=self.src_dir, env=env)
# We generally prefer to call the `autogen.sh` script provided
# by the project on the basis that it may know more than plain
# autoreconf does.
if os.path.exists(autogen_path):
self._run_cmd(["bash", autogen_path], cwd=self.src_dir, env=env)
else:
self._run_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env)
configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args
self._run_cmd(configure_cmd, env=env)
only_install = self.manifest.get("build", "only_install", "false", ctx=self.ctx)
if not only_install:
self._run_cmd([self._make_binary, "-j%s" % self.num_jobs], env=env)
self._run_cmd([self._make_binary, "install"], env=env)
class Iproute2Builder(BuilderBase):
# ./configure --prefix does not work for iproute2.
# Thus, explicitly copy sources from src_dir to build_dir, build,
# and then install to inst_dir using DESTDIR
# lastly, also copy include from build_dir to inst_dir
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) -> None:
super(Iproute2Builder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure) -> None:
configure_path = os.path.join(self.src_dir, "configure")
env = self.env.copy()
self._run_cmd([configure_path], env=env)
shutil.rmtree(self.build_dir)
shutil.copytree(self.src_dir, self.build_dir)
self._run_cmd(["make", "-j%s" % self.num_jobs], env=env)
install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir]
for d in ["include", "lib"]:
if not os.path.isdir(os.path.join(self.inst_dir, d)):
shutil.copytree(
os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d)
)
self._run_cmd(install_cmd, env=env)
class CMakeBuilder(BuilderBase):
MANUAL_BUILD_SCRIPT = """\
#!{sys.executable}
import argparse
import subprocess
import sys
CMAKE = {cmake!r}
CTEST = {ctest!r}
SRC_DIR = {src_dir!r}
BUILD_DIR = {build_dir!r}
INSTALL_DIR = {install_dir!r}
CMD_PREFIX = {cmd_prefix!r}
CMAKE_ENV = {env_str}
CMAKE_DEFINE_ARGS = {define_args_str}
def get_jobs_argument(num_jobs_arg: int) -> str:
if num_jobs_arg > 0:
return "-j" + str(num_jobs_arg)
import multiprocessing
num_jobs = multiprocessing.cpu_count() // 2
return "-j" + str(num_jobs)
def main():
ap = argparse.ArgumentParser()
ap.add_argument(
"cmake_args",
nargs=argparse.REMAINDER,
help='Any extra arguments after an "--" argument will be passed '
"directly to CMake."
)
ap.add_argument(
"--mode",
choices=["configure", "build", "install", "test"],
default="configure",
help="The mode to run: configure, build, or install. "
"Defaults to configure",
)
ap.add_argument(
"--build",
action="store_const",
const="build",
dest="mode",
help="An alias for --mode=build",
)
ap.add_argument(
"-j",
"--num-jobs",
action="store",
type=int,
default=0,
help="Run the build or tests with the specified number of parallel jobs",
)
ap.add_argument(
"--install",
action="store_const",
const="install",
dest="mode",
help="An alias for --mode=install",
)
ap.add_argument(
"--test",
action="store_const",
const="test",
dest="mode",
help="An alias for --mode=test",
)
args = ap.parse_args()
# Strip off a leading "--" from the additional CMake arguments
if args.cmake_args and args.cmake_args[0] == "--":
args.cmake_args = args.cmake_args[1:]
env = CMAKE_ENV
if args.mode == "configure":
full_cmd = CMD_PREFIX + [CMAKE, SRC_DIR] + CMAKE_DEFINE_ARGS + args.cmake_args
elif args.mode in ("build", "install"):
target = "all" if args.mode == "build" else "install"
full_cmd = CMD_PREFIX + [
CMAKE,
"--build",
BUILD_DIR,
"--target",
target,
"--config",
"{build_type}",
get_jobs_argument(args.num_jobs),
] + args.cmake_args
elif args.mode == "test":
full_cmd = CMD_PREFIX + [
{dev_run_script}CTEST,
"--output-on-failure",
get_jobs_argument(args.num_jobs),
] + args.cmake_args
else:
ap.error("unknown invocation mode: %s" % (args.mode,))
cmd_str = " ".join(full_cmd)
print("Running: %r" % (cmd_str,))
proc = subprocess.run(full_cmd, env=env, cwd=BUILD_DIR)
sys.exit(proc.returncode)
if __name__ == "__main__":
main()
"""
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
defines,
final_install_prefix=None,
extra_cmake_defines=None,
cmake_target="install",
) -> None:
super(CMakeBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
final_install_prefix=final_install_prefix,
)
self.defines = defines or {}
if extra_cmake_defines:
self.defines.update(extra_cmake_defines)
self.cmake_target = cmake_target
try:
from .facebook.vcvarsall import extra_vc_cmake_defines
except ImportError:
pass
else:
self.defines.update(extra_vc_cmake_defines)
self.loader = loader
if build_opts.shared_libs:
self.defines["BUILD_SHARED_LIBS"] = "ON"
def _invalidate_cache(self) -> None:
for name in [
"CMakeCache.txt",
"CMakeFiles/CMakeError.log",
"CMakeFiles/CMakeOutput.log",
]:
name = os.path.join(self.build_dir, name)
if os.path.isdir(name):
shutil.rmtree(name)
elif os.path.exists(name):
os.unlink(name)
def _needs_reconfigure(self) -> bool:
for name in ["CMakeCache.txt", "build.ninja"]:
name = os.path.join(self.build_dir, name)
if not os.path.exists(name):
return True
return False
def _write_build_script(self, **kwargs) -> None:
env_lines = [" {!r}: {!r},".format(k, v) for k, v in kwargs["env"].items()]
kwargs["env_str"] = "\n".join(["{"] + env_lines + ["}"])
if self.build_opts.is_windows():
kwargs["dev_run_script"] = '"powershell.exe", {!r}, '.format(
self.get_dev_run_script_path()
)
else:
kwargs["dev_run_script"] = ""
define_arg_lines = ["["]
for arg in kwargs["define_args"]:
# Replace the CMAKE_INSTALL_PREFIX argument to use the INSTALL_DIR
# variable that we define in the MANUAL_BUILD_SCRIPT code.
if arg.startswith("-DCMAKE_INSTALL_PREFIX="):
value = " {!r}.format(INSTALL_DIR),".format(
"-DCMAKE_INSTALL_PREFIX={}"
)
else:
value = " {!r},".format(arg)
define_arg_lines.append(value)
define_arg_lines.append("]")
kwargs["define_args_str"] = "\n".join(define_arg_lines)
# In order to make it easier for developers to manually run builds for
# CMake-based projects, write out some build scripts that can be used to invoke
# CMake manually.
build_script_path = os.path.join(self.build_dir, "run_cmake.py")
script_contents = self.MANUAL_BUILD_SCRIPT.format(**kwargs)
with open(build_script_path, "wb") as f:
f.write(script_contents.encode())
os.chmod(build_script_path, 0o755)
def _compute_cmake_define_args(self, env):
defines = {
"CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir,
"BUILD_SHARED_LIBS": "OFF",
# Some of the deps (rsocket) default to UBSAN enabled if left
# unspecified. Some of the deps fail to compile in release mode
# due to warning->error promotion. RelWithDebInfo is the happy
# medium.
"CMAKE_BUILD_TYPE": self.build_opts.build_type,
}
if "SANDCASTLE" not in os.environ:
# We sometimes see intermittent ccache related breakages on some
# of the FB internal CI hosts, so we prefer to disable ccache
# when running in that environment.
ccache = path_search(env, "ccache")
if ccache:
defines["CMAKE_CXX_COMPILER_LAUNCHER"] = ccache
else:
# rocksdb does its own probing for ccache.
# Ensure that it is disabled on sandcastle
env["CCACHE_DISABLE"] = "1"
# Some sandcastle hosts have broken ccache related dirs, and
# even though we've asked for it to be disabled ccache is
# still invoked by rocksdb's cmake.
# Redirect its config directory to somewhere that is guaranteed
# fresh to us, and that won't have any ccache data inside.
env["CCACHE_DIR"] = f"{self.build_opts.scratch_dir}/ccache"
if "GITHUB_ACTIONS" in os.environ and self.build_opts.is_windows():
# GitHub actions: the host has both gcc and msvc installed, and
# the default behavior of cmake is to prefer gcc.
# Instruct cmake that we want it to use cl.exe; this is important
# because Boost prefers cl.exe and the mismatch results in cmake
# with gcc not being able to find boost built with cl.exe.
defines["CMAKE_C_COMPILER"] = "cl.exe"
defines["CMAKE_CXX_COMPILER"] = "cl.exe"
if self.build_opts.is_darwin():
# Try to persuade cmake to set the rpath to match the lib
# dirs of the dependencies. This isn't automatic, and to
# make things more interesting, cmake uses `;` as the path
# separator, so translate the runtime path to something
# that cmake will parse
defines["CMAKE_INSTALL_RPATH"] = ";".join(
env.get("DYLD_LIBRARY_PATH", "").split(":")
)
# Tell cmake that we want to set the rpath in the tree
# at build time. Without this the rpath is only set
# at the moment that the binaries are installed. That
# default is problematic for example when using the
# gtest integration in cmake which runs the built test
# executables during the build to discover the set of
# tests.
defines["CMAKE_BUILD_WITH_INSTALL_RPATH"] = "ON"
boost_169_is_required = False
if self.loader:
for m in self.loader.manifests_in_dependency_order():
preinstalled = m.get_section_as_dict("preinstalled.env", self.ctx)
boost_169_is_required = "BOOST_ROOT_1_69_0" in preinstalled.keys()
if boost_169_is_required:
break
if (
boost_169_is_required
and self.build_opts.allow_system_packages
and self.build_opts.host_type.get_package_manager()
and self.build_opts.host_type.get_package_manager() == "rpm"
):
# Boost 1.69 rpms don't install cmake config to the system, so to point to them explicitly
defines["BOOST_INCLUDEDIR"] = "/usr/include/boost169"
defines["BOOST_LIBRARYDIR"] = "/usr/lib64/boost169"
defines.update(self.defines)
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
# if self.build_opts.is_windows():
# define_args += ["-G", "Visual Studio 15 2017 Win64"]
define_args += ["-G", "Ninja"]
return define_args
def _build(self, reconfigure: bool) -> None:
reconfigure = reconfigure or self._needs_reconfigure()
env = self._compute_env()
if not self.build_opts.is_windows() and self.final_install_prefix:
env["DESTDIR"] = self.inst_dir
# Resolve the cmake that we installed
cmake = path_search(env, "cmake")
if cmake is None:
raise Exception("Failed to find CMake")
if reconfigure:
define_args = self._compute_cmake_define_args(env)
self._write_build_script(
cmd_prefix=self._get_cmd_prefix(),
cmake=cmake,
ctest=path_search(env, "ctest"),
env=env,
define_args=define_args,
src_dir=self.src_dir,
build_dir=self.build_dir,
install_dir=self.inst_dir,
sys=sys,
build_type=self.build_opts.build_type,
)
self._invalidate_cache()
self._run_cmd([cmake, self.src_dir] + define_args, env=env)
self._run_cmd(
[
cmake,
"--build",
self.build_dir,
"--target",
self.cmake_target,
"--config",
self.build_opts.build_type,
"-j",
str(self.num_jobs),
],
env=env,
)
def run_tests(
self, schedule_type, owner, test_filter, retry: int, no_testpilot
) -> None:
env = self._compute_env()
ctest = path_search(env, "ctest")
cmake = path_search(env, "cmake")
def require_command(path: Optional[str], name: str) -> str:
if path is None:
raise RuntimeError("unable to find command `{}`".format(name))
return path
# On Windows, we also need to update $PATH to include the directories that
# contain runtime library dependencies. This is not needed on other platforms
# since CMake will emit RPATH properly in the binary so they can find these
# dependencies.
if self.build_opts.is_windows():
path_entries = self.get_dev_run_extra_path_dirs()
path = env.get("PATH")
if path:
path_entries.insert(0, path)
env["PATH"] = ";".join(path_entries)
# Don't use the cmd_prefix when running tests. This is vcvarsall.bat on
# Windows. vcvarsall.bat is only needed for the build, not tests. It
# unfortunately fails if invoked with a long PATH environment variable when
# running the tests.
use_cmd_prefix = False
def get_property(test, propname, defval=None):
"""extracts a named property from a cmake test info json blob.
The properties look like:
[{"name": "WORKING_DIRECTORY"},
{"value": "something"}]
We assume that it is invalid for the same named property to be
listed more than once.
"""
props = test.get("properties", [])
for p in props:
if p.get("name", None) == propname:
return p.get("value", defval)
return defval
def list_tests():
output = subprocess.check_output(
[require_command(ctest, "ctest"), "--show-only=json-v1"],
env=env,
cwd=self.build_dir,
)
try:
data = json.loads(output.decode("utf-8"))
except ValueError as exc:
raise Exception(
"Failed to decode cmake test info using %s: %s. Output was: %r"
% (ctest, str(exc), output)
)
tests = []
machine_suffix = self.build_opts.host_type.as_tuple_string()
for test in data["tests"]:
working_dir = get_property(test, "WORKING_DIRECTORY")
labels = []
machine_suffix = self.build_opts.host_type.as_tuple_string()
labels.append("tpx-fb-test-type=3")
labels.append("tpx_test_config::buildsystem=getdeps")
labels.append("tpx_test_config::platform={}".format(machine_suffix))
if get_property(test, "DISABLED"):
labels.append("disabled")
command = test["command"]
if working_dir:
command = [
require_command(cmake, "cmake"),
"-E",
"chdir",
working_dir,
] + command
import os
tests.append(
{
"type": "custom",
"target": "%s-%s-getdeps-%s"
% (self.manifest.name, test["name"], machine_suffix),
"command": command,
"labels": labels,
"env": {},
"required_paths": [],
"contacts": [],
"cwd": os.getcwd(),
}
)
return tests
if schedule_type == "continuous" or schedule_type == "testwarden":
# for continuous and testwarden runs, disabling retry can give up
# better signals for flaky tests.
retry = 0
tpx = path_search(env, "tpx")
if tpx and not no_testpilot:
buck_test_info = list_tests()
import os
from .facebook.testinfra import start_run
buck_test_info_name = os.path.join(self.build_dir, ".buck-test-info.json")
with open(buck_test_info_name, "w") as f:
json.dump(buck_test_info, f)
env.set("http_proxy", "")
env.set("https_proxy", "")
runs = []
with start_run(env["FBSOURCE_HASH"]) as run_id:
testpilot_args = [
tpx,
"--force-local-execution",
"--buck-test-info",
buck_test_info_name,
"--retry=%d" % retry,
"-j=%s" % str(self.num_jobs),
"--print-long-results",
]
if owner:
testpilot_args += ["--contacts", owner]
if env:
testpilot_args.append("--env")
testpilot_args.extend(f"{key}={val}" for key, val in env.items())
if run_id is not None:
testpilot_args += ["--run-id", run_id]
if test_filter:
testpilot_args += ["--", test_filter]
if schedule_type == "diff":
runs.append(["--collection", "oss-diff", "--purpose", "diff"])
elif schedule_type == "continuous":
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-continuous",
"--purpose",
"continuous",
]
)
elif schedule_type == "testwarden":
# One run to assess new tests
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-new-test-stress",
"--stress-runs",
"10",
"--purpose",
"stress-run-new-test",
]
)
# And another for existing tests
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-existing-test-stress",
"--stress-runs",
"10",
"--purpose",
"stress-run",
]
)
else:
runs.append([])
for run in runs:
self._run_cmd(
testpilot_args + run,
cwd=self.build_opts.fbcode_builder_dir,
env=env,
use_cmd_prefix=use_cmd_prefix,
)
else:
args = [
require_command(ctest, "ctest"),
"--output-on-failure",
"-j",
str(self.num_jobs),
]
if test_filter:
args += ["-R", test_filter]
count = 0
while count <= retry:
retcode = self._run_cmd(
args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True
)
if retcode == 0:
break
if count == 0:
# Only add this option in the second run.
args += ["--rerun-failed"]
count += 1
# pyre-fixme[61]: `retcode` is undefined, or not always defined.
if retcode != 0:
# Allow except clause in getdeps.main to catch and exit gracefully
# This allows non-testpilot runs to fail through the same logic as failed testpilot runs, which may become handy in case if post test processing is needed in the future
# pyre-fixme[61]: `retcode` is undefined, or not always defined.
raise subprocess.CalledProcessError(retcode, args)
class NinjaBootstrap(BuilderBase):
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
build_dir,
src_dir,
inst_dir,
) -> None:
super(NinjaBootstrap, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure) -> None:
self._run_cmd([sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir)
src_ninja = os.path.join(self.src_dir, "ninja")
dest_ninja = os.path.join(self.inst_dir, "bin/ninja")
bin_dir = os.path.dirname(dest_ninja)
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
shutil.copyfile(src_ninja, dest_ninja)
shutil.copymode(src_ninja, dest_ninja)
class OpenSSLBuilder(BuilderBase):
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
build_dir,
src_dir,
inst_dir,
) -> None:
super(OpenSSLBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure) -> None:
configure = os.path.join(self.src_dir, "Configure")
# prefer to resolve the perl that we installed from
# our manifest on windows, but fall back to the system
# path on eg: darwin
env = self.env.copy()
for m in self.dep_manifests:
bindir = os.path.join(self.loader.get_project_install_dir(m), "bin")
add_path_entry(env, "PATH", bindir, append=False)
perl = typing.cast(str, path_search(env, "perl", "perl"))
make_j_args = []
if self.build_opts.is_windows():
make = "nmake.exe"
args = ["VC-WIN64A-masm", "-utf-8"]
elif self.build_opts.is_darwin():
make = "make"
make_j_args = ["-j%s" % self.num_jobs]
args = (
["darwin64-x86_64-cc"]
if not self.build_opts.is_arm()
else ["darwin64-arm64-cc"]
)
elif self.build_opts.is_linux():
make = "make"
make_j_args = ["-j%s" % self.num_jobs]
args = (
["linux-x86_64"] if not self.build_opts.is_arm() else ["linux-aarch64"]
)
else:
raise Exception("don't know how to build openssl for %r" % self.ctx)
self._run_cmd(
[
perl,
configure,
"--prefix=%s" % self.inst_dir,
"--openssldir=%s" % self.inst_dir,
]
+ args
+ [
"enable-static-engine",
"enable-capieng",
"no-makedepend",
"no-unit-test",
"no-tests",
]
)
make_build = [make] + make_j_args
self._run_cmd(make_build)
make_install = [make, "install_sw", "install_ssldirs"]
self._run_cmd(make_install)
class Boost(BuilderBase):
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
b2_args,
) -> None:
children = os.listdir(src_dir)
assert len(children) == 1, "expected a single directory entry: %r" % (children,)
boost_src = children[0]
assert boost_src.startswith("boost")
src_dir = os.path.join(src_dir, children[0])
super(Boost, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
self.b2_args = b2_args
def _build(self, reconfigure) -> None:
env = self._compute_env()
linkage = ["static"]
if self.build_opts.is_windows() or self.build_opts.shared_libs:
linkage.append("shared")
args = []
if self.build_opts.is_darwin():
clang = subprocess.check_output(["xcrun", "--find", "clang"])
user_config = os.path.join(self.build_dir, "project-config.jam")
with open(user_config, "w") as jamfile:
jamfile.write("using clang : : %s ;\n" % clang.decode().strip())
args.append("--user-config=%s" % user_config)
for link in linkage:
bootstrap_args = self.manifest.get_section_as_args(
"bootstrap.args", self.ctx
)
if self.build_opts.is_windows():
bootstrap = os.path.join(self.src_dir, "bootstrap.bat")
self._run_cmd([bootstrap] + bootstrap_args, cwd=self.src_dir, env=env)
args += ["address-model=64"]
else:
bootstrap = os.path.join(self.src_dir, "bootstrap.sh")
self._run_cmd(
[bootstrap, "--prefix=%s" % self.inst_dir] + bootstrap_args,
cwd=self.src_dir,
env=env,
)
b2 = os.path.join(self.src_dir, "b2")
self._run_cmd(
[
b2,
"-j%s" % self.num_jobs,
"--prefix=%s" % self.inst_dir,
"--builddir=%s" % self.build_dir,
]
+ args
+ self.b2_args
+ [
"link=%s" % link,
"runtime-link=shared",
"variant=release",
"threading=multi",
"debug-symbols=on",
"visibility=global",
"-d2",
"install",
],
cwd=self.src_dir,
env=env,
)
class NopBuilder(BuilderBase):
def __init__(
self, loader, dep_manifests, build_opts, ctx, manifest, src_dir, inst_dir
) -> None:
super(NopBuilder, self).__init__(
loader, dep_manifests, build_opts, ctx, manifest, src_dir, None, inst_dir
)
def build(self, reconfigure: bool) -> None:
print("Installing %s -> %s" % (self.src_dir, self.inst_dir))
parent = os.path.dirname(self.inst_dir)
if not os.path.exists(parent):
os.makedirs(parent)
install_files = self.manifest.get_section_as_ordered_pairs(
"install.files", self.ctx
)
if install_files:
for src_name, dest_name in self.manifest.get_section_as_ordered_pairs(
"install.files", self.ctx
):
full_dest = os.path.join(self.inst_dir, dest_name)
full_src = os.path.join(self.src_dir, src_name)
dest_parent = os.path.dirname(full_dest)
if not os.path.exists(dest_parent):
os.makedirs(dest_parent)
if os.path.isdir(full_src):
if not os.path.exists(full_dest):
shutil.copytree(full_src, full_dest)
else:
shutil.copyfile(full_src, full_dest)
shutil.copymode(full_src, full_dest)
# This is a bit gross, but the mac ninja.zip doesn't
# give ninja execute permissions, so force them on
# for things that look like they live in a bin dir
if os.path.dirname(dest_name) == "bin":
st = os.lstat(full_dest)
os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
else:
if not os.path.exists(self.inst_dir):
shutil.copytree(self.src_dir, self.inst_dir)
class SqliteBuilder(BuilderBase):
def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) -> None:
super(SqliteBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure) -> None:
for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]:
src = os.path.join(self.src_dir, f)
dest = os.path.join(self.build_dir, f)
copy_if_different(src, dest)
cmake_lists = """
cmake_minimum_required(VERSION 3.1.3 FATAL_ERROR)
project(sqlite3 C)
add_library(sqlite3 STATIC sqlite3.c)
# These options are taken from the defaults in Makefile.msc in
# the sqlite distribution
target_compile_definitions(sqlite3 PRIVATE
-DSQLITE_ENABLE_COLUMN_METADATA=1
-DSQLITE_ENABLE_FTS3=1
-DSQLITE_ENABLE_RTREE=1
-DSQLITE_ENABLE_GEOPOLY=1
-DSQLITE_ENABLE_JSON1=1
-DSQLITE_ENABLE_STMTVTAB=1
-DSQLITE_ENABLE_DBPAGE_VTAB=1
-DSQLITE_ENABLE_DBSTAT_VTAB=1
-DSQLITE_INTROSPECTION_PRAGMAS=1
-DSQLITE_ENABLE_DESERIALIZE=1
)
install(TARGETS sqlite3)
install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
"""
with open(os.path.join(self.build_dir, "CMakeLists.txt"), "w") as f:
f.write(cmake_lists)
defines = {
"CMAKE_INSTALL_PREFIX": self.inst_dir,
"BUILD_SHARED_LIBS": "ON" if self.build_opts.shared_libs else "OFF",
"CMAKE_BUILD_TYPE": "RelWithDebInfo",
}
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
define_args += ["-G", "Ninja"]
env = self._compute_env()
# Resolve the cmake that we installed
cmake = path_search(env, "cmake")
self._run_cmd([cmake, self.build_dir] + define_args, env=env)
self._run_cmd(
[
cmake,
"--build",
self.build_dir,
"--target",
"install",
"--config",
self.build_opts.build_type,
"-j",
str(self.num_jobs),
],
env=env,
)
Codebase Browser
folly