test_launcher.cc | Explore in Territory

// Copyright 2013 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/40284755): Remove this and spanify to fix the errors.
#pragma allow_unsafe_buffers
#endif

#include "base/test/launcher/test_launcher.h"

#include <stdio.h>

#include <algorithm>
#include <map>
#include <random>
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <utility>

#include "base/at_exit.h"
#include "base/clang_profiling_buildflags.h"
#include "base/command_line.h"
#include "base/containers/adapters.h"
#include "base/containers/contains.h"
#include "base/environment.h"
#include "base/files/file_enumerator.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/files/scoped_file.h"
#include "base/files/scoped_temp_dir.h"
#include "base/format_macros.h"
#include "base/functional/bind.h"
#include "base/hash/hash.h"
#include "base/lazy_instance.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/memory/raw_ptr.h"
#include "base/memory/raw_ref.h"
#include "base/numerics/safe_conversions.h"
#include "base/process/kill.h"
#include "base/process/launch.h"
#include "base/ranges/algorithm.h"
#include "base/run_loop.h"
#include "base/strings/pattern.h"
#include "base/strings/strcat.h"
#include "base/strings/string_number_conversions.h"

#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/strings/stringize_macros.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "base/system/sys_info.h"
#include "base/task/post_job.h"
#include "base/task/single_thread_task_runner.h"
#include "base/task/thread_pool.h"
#include "base/task/thread_pool/thread_pool_instance.h"
#include "base/test/gtest_util.h"
#include "base/test/gtest_xml_util.h"
#include "base/test/launcher/test_launcher_tracer.h"
#include "base/test/launcher/test_results_tracker.h"
#include "base/test/scoped_logging_settings.h"
#include "base/test/test_file_util.h"
#include "base/test/test_switches.h"
#include "base/test/test_timeouts.h"
#include "base/threading/platform_thread.h"
#include "base/threading/thread_restrictions.h"
#include "base/time/time.h"
#include "build/build_config.h"
#include "build/chromeos_buildflags.h"
#include "testing/gtest/include/gtest/gtest.h"

#if BUILDFLAG(IS_POSIX)
#include <fcntl.h>

#include "base/files/file_descriptor_watcher_posix.h"
#endif

#if BUILDFLAG(IS_APPLE)
#include "base/apple/scoped_nsautorelease_pool.h"
#endif

#if BUILDFLAG(IS_WIN)
#include <windows.h>

#include "base/strings/string_util_win.h"

// To avoid conflicts with the macro from the Windows SDK...
#undef GetCommandLine
#endif

#if BUILDFLAG(IS_FUCHSIA)
#include <lib/fdio/namespace.h>
#include <lib/zx/job.h>
#include <lib/zx/time.h>
#include "base/atomic_sequence_num.h"
#include "base/fuchsia/default_job.h"
#include "base/fuchsia/file_utils.h"
#include "base/fuchsia/fuchsia_logging.h"
#endif

#if BUILDFLAG(IS_IOS)
#include "base/path_service.h"
#endif

namespace base {

// See
// https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ
operator<<;

// The environment variable name for the total number of test shards.
const char kTestTotalShards[] = …;
// The environment variable name for the test shard index.
const char kTestShardIndex[] = …;

// Prefix indicating test has to run prior to the other test.
const char kPreTestPrefix[] = …;

// Prefix indicating test is disabled, will not run unless specified.
const char kDisabledTestPrefix[] = …;

ResultWatcher::ResultWatcher(FilePath result_file, size_t num_tests)
    : … { … }

bool ResultWatcher::PollUntilDone(TimeDelta timeout_per_test) { … }

TimeDelta ResultWatcher::PollOnce(TimeDelta timeout_per_test) { … }

Time ResultWatcher::LatestCompletionTimestamp(
    const std::vector<TestResult>& test_results) { … }

// Watch results generated by a child test process. Wait for the child process
// to exit between result checks.
class ProcessResultWatcher : public ResultWatcher { … };

int ProcessResultWatcher::GetExitCode() { … }

bool ProcessResultWatcher::WaitWithTimeout(TimeDelta timeout) { … }

namespace {

// Global tag for test runs where the results are unreliable for any reason.
const char kUnreliableResultsTag[] = …;

// Maximum time of no output after which we print list of processes still
// running. This deliberately doesn't use TestTimeouts (which is otherwise
// a recommended solution), because they can be increased. This would defeat
// the purpose of this timeout, which is 1) to avoid buildbot "no output for
// X seconds" timeout killing the process 2) help communicate status of
// the test launcher to people looking at the output (no output for a long
// time is mysterious and gives no info about what is happening) 3) help
// debugging in case the process hangs anyway.
constexpr TimeDelta kOutputTimeout = …;

// Limit of output snippet lines when printing to stdout.
// Avoids flooding the logs with amount of output that gums up
// the infrastructure.
const size_t kOutputSnippetLinesLimit = …;

// Limit of output snippet size. Exceeding this limit
// results in truncating the output and failing the test.
const size_t kOutputSnippetBytesLimit = …;

// Limit of seed values for gtest shuffling. Arbitrary, but based on
// gtest's similarly arbitrary choice.
const uint32_t kRandomSeedUpperBound = …;

// Set of live launch test processes with corresponding lock (it is allowed
// for callers to launch processes on different threads).
Lock* GetLiveProcessesLock() { … }

std::map<ProcessHandle, CommandLine>* GetLiveProcesses() { … }

// Performance trace generator.
TestLauncherTracer* GetTestLauncherTracer() { … }

#if BUILDFLAG(IS_FUCHSIA)
zx_status_t WaitForJobExit(const zx::job& job) {
  zx::time deadline =
      zx::deadline_after(zx::duration(kOutputTimeout.ToZxDuration()));
  zx_signals_t to_wait_for = ZX_JOB_NO_JOBS | ZX_JOB_NO_PROCESSES;
  while (to_wait_for) {
    zx_signals_t observed = 0;
    zx_status_t status = job.wait_one(to_wait_for, deadline, &observed);
    if (status != ZX_OK)
      return status;
    to_wait_for &= ~observed;
  }
  return ZX_OK;
}
#endif  // BUILDFLAG(IS_FUCHSIA)

#if BUILDFLAG(IS_POSIX)
// Self-pipe that makes it possible to do complex shutdown handling
// outside of the signal handler.
int g_shutdown_pipe[2] = …;

void ShutdownPipeSignalHandler(int signal) { … }

void KillSpawnedTestProcesses() { … }
#endif  // BUILDFLAG(IS_POSIX)

// Parses the environment variable var as an Int32.  If it is unset, returns
// true.  If it is set, unsets it then converts it to Int32 before
// returning it in |result|.  Returns true on success.
bool TakeInt32FromEnvironment(const char* const var, int32_t* result) { … }

// Unsets the environment variable |name| and returns true on success.
// Also returns true if the variable just doesn't exist.
bool UnsetEnvironmentVariableIfExists(const std::string& name) { … }

// Returns true if bot mode has been requested, i.e. defaults optimized
// for continuous integration bots. This way developers don't have to remember
// special command-line flags.
bool BotModeEnabled(const CommandLine* command_line) { … }

// Returns command line command line after gtest-specific processing
// and applying |wrapper|.
CommandLine PrepareCommandLineForGTest(const CommandLine& command_line,
                                       const std::string& wrapper,
                                       const size_t retries_left) { … }

// Launches a child process using |command_line|. If a test is still running
// after |timeout|, the child process is terminated and |*was_timeout| is set to
// true. Returns exit code of the process.
int LaunchChildTestProcessWithOptions(const CommandLine& command_line,
                                      const LaunchOptions& options,
                                      int flags,
                                      const FilePath& result_file,
                                      TimeDelta timeout_per_test,
                                      size_t num_tests,
                                      TestLauncherDelegate* delegate,
                                      bool* was_timeout) { … }

struct ChildProcessResults { … };

// Returns the path to a temporary directory within |task_temp_dir| for the
// child process of index |child_index|, or an empty FilePath if per-child temp
// dirs are not supported.
FilePath CreateChildTempDirIfSupported(const FilePath& task_temp_dir,
                                       int child_index) { … }

// Adds the platform-specific variable setting |temp_dir| as a process's
// temporary directory to |environment|.
void SetTemporaryDirectory(const FilePath& temp_dir,
                           EnvironmentMap* environment) { … }

// This launches the child test process, waits for it to complete,
// and returns child process results.
ChildProcessResults DoLaunchChildTestProcess(
    const CommandLine& command_line,
    const FilePath& process_temp_dir,
    const FilePath& result_file,
    TimeDelta timeout_per_test,
    size_t num_tests,
    const TestLauncher::LaunchOptions& test_launch_options,
    bool redirect_stdio,
    TestLauncherDelegate* delegate) { … }

std::vector<std::string> ExtractTestsFromFilter(const std::string& filter,
                                                bool double_colon_supported) { … }

// A test runner object to run tests across a number of sequence runners,
// and control running pre tests in sequence.
class TestRunner { … };

void TestRunner::Run(const std::vector<std::string>& test_names) { … }

void TestRunner::WorkerTask(scoped_refptr<TaskRunner> main_task_runner,
                            base::JobDelegate* delegate) { … }

void TestRunner::CleanupTask(base::ScopedTempDir task_temp_dir, bool done) { … }

// Returns the number of files and directories in |dir|, or 0 if |dir| is empty.
int CountItemsInDirectory(const FilePath& dir) { … }

// Truncates a snippet in the middle to the given byte limit. byte_limit should
// be at least 30.
std::string TruncateSnippet(const std::string_view snippet, size_t byte_limit) { … }

}  // namespace

const char kGTestBreakOnFailure[] = …;
const char kGTestFilterFlag[] = …;
const char kGTestFlagfileFlag[] = …;
const char kGTestHelpFlag[]   = …;
const char kGTestListTestsFlag[] = …;
const char kGTestRepeatFlag[] = …;
const char kGTestRunDisabledTestsFlag[] = …;
const char kGTestOutputFlag[] = …;
const char kGTestShuffleFlag[] = …;
const char kGTestRandomSeedFlag[] = …;
const char kIsolatedScriptRunDisabledTestsFlag[] = …;
const char kIsolatedScriptTestFilterFlag[] = …;
const char kIsolatedScriptTestRepeatFlag[] = …;

class TestLauncher::TestInfo { … };

TestLauncher::TestInfo::TestInfo(const TestIdentifier& test_id)
    : … { … }

std::string TestLauncher::TestInfo::GetDisabledStrippedName() const { … }

std::string TestLauncher::TestInfo::GetFullName() const { … }

std::string TestLauncher::TestInfo::GetPreName() const { … }

std::string TestLauncher::TestInfo::GetPrefixStrippedName() const { … }

TestLauncherDelegate::~TestLauncherDelegate() = default;

bool TestLauncherDelegate::ShouldRunTest(const TestIdentifier& test) { … }

TestLauncher::LaunchOptions::LaunchOptions() = default;
TestLauncher::LaunchOptions::LaunchOptions(const LaunchOptions& other) =
    default;
TestLauncher::LaunchOptions::~LaunchOptions() = default;

TestLauncher::TestLauncher(TestLauncherDelegate* launcher_delegate,
                           size_t parallel_jobs,
                           size_t retry_limit)
    : … { … }

TestLauncher::~TestLauncher() { … }

bool TestLauncher::Run(CommandLine* command_line) { … }

void TestLauncher::LaunchChildGTestProcess(
    scoped_refptr<TaskRunner> task_runner,
    const std::vector<std::string>& test_names,
    const FilePath& task_temp_dir,
    const FilePath& child_temp_dir) { … }

// Determines which result status will be assigned for missing test results.
TestResult::Status MissingResultStatus(size_t tests_to_run_count,
                                       bool was_timeout,
                                       bool exit_code) { … }

// Returns interpreted test results.
void TestLauncher::ProcessTestResults(
    const std::vector<std::string>& test_names,
    const FilePath& result_file,
    const std::string& output,
    TimeDelta elapsed_time,
    int exit_code,
    bool was_timeout,
    PlatformThreadId thread_id,
    int process_num,
    int leaked_items) { … }

void TestLauncher::OnTestFinished(const TestResult& original_result) { … }

// Helper used to parse test filter files. Syntax is documented in
// //testing/buildbot/filters/README.md .
bool LoadFilterFile(const FilePath& file_path,
                    std::vector<std::string>* positive_filter,
                    std::vector<std::string>* negative_filter) { … }

bool TestLauncher::IsOnlyExactPositiveFilterFromFile(
    const CommandLine* command_line) const { … }

bool TestLauncher::Init(CommandLine* command_line) { … }

bool TestLauncher::InitTests() { … }

bool TestLauncher::ShuffleTests(CommandLine* command_line) { … }

bool TestLauncher::ProcessAndValidateTests() { … }

void TestLauncher::CreateAndStartThreadPool(size_t num_parallel_jobs) { … }

void TestLauncher::CombinePositiveTestFilters(
    std::vector<std::string> filter_a,
    std::vector<std::string> filter_b) { … }

bool TestLauncher::ShouldRunInCurrentShard(
    std::string_view prefix_stripped_name) const { … }

std::vector<std::string> TestLauncher::CollectTests() { … }

void TestLauncher::RunTests() { … }

void TestLauncher::PrintFuzzyMatchingTestNames() { … }

bool TestLauncher::RunRetryTests() { … }

void TestLauncher::OnTestIterationStart() { … }

#if BUILDFLAG(IS_POSIX)
// I/O watcher for the reading end of the self-pipe above.
// Terminates any launched child processes and exits the process.
void TestLauncher::OnShutdownPipeReadable() { … }
#endif  // BUILDFLAG(IS_POSIX)

void TestLauncher::MaybeSaveSummaryAsJSON(
    const std::vector<std::string>& additional_tags) { … }

void TestLauncher::OnTestIterationFinished() { … }

void TestLauncher::OnOutputTimeout() { … }

size_t NumParallelJobs(unsigned int cores_per_job) { … }

std::string GetTestOutputSnippet(const TestResult& result,
                                 const std::string& full_output) { … }

std::string TruncateSnippetFocused(const std::string_view snippet,
                                   size_t byte_limit) { … }

}  // namespace base
chromium/base/test/launcher/test_launcher.cc
