test_launcher_unittest.cc | Explore in Territory

// Copyright 2019 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/test/launcher/test_launcher.h"

#include <stddef.h>

#include <optional>

#include "base/base64.h"
#include "base/command_line.h"
#include "base/files/file_util.h"
#include "base/files/scoped_temp_dir.h"
#include "base/functional/bind.h"
#include "base/functional/callback_helpers.h"
#include "base/i18n/time_formatting.h"
#include "base/logging.h"
#include "base/no_destructor.h"
#include "base/process/launch.h"
#include "base/strings/strcat.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/test/gtest_xml_util.h"
#include "base/test/launcher/test_launcher_test_utils.h"
#include "base/test/launcher/unit_test_launcher.h"
#include "base/test/multiprocess_test.h"
#include "base/test/scoped_logging_settings.h"
#include "base/test/task_environment.h"
#include "base/test/test_timeouts.h"
#include "build/build_config.h"
#include "build/chromeos_buildflags.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "testing/multiprocess_func_list.h"

namespace base {
namespace {

_;
DoAll;
Invoke;
InvokeWithoutArgs;
Return;
ReturnPointee;

TestResult GenerateTestResult(const std::string& test_name,
                              TestResult::Status status,
                              TimeDelta elapsed_td = Milliseconds(30),
                              const std::string& output_snippet = "output") { … }

TestResultPart GenerateTestResultPart(TestResultPart::Type type,
                                      const std::string& file_name,
                                      int line_number,
                                      const std::string& summary,
                                      const std::string& message) { … }

// Mock TestLauncher to mock CreateAndStartThreadPool,
// unit test will provide a TaskEnvironment.
class MockTestLauncher : public TestLauncher { … };

// Simple TestLauncherDelegate mock to test TestLauncher flow.
class MockTestLauncherDelegate : public TestLauncherDelegate { … };

class MockResultWatcher : public ResultWatcher { … };

// Using MockTestLauncher to test TestLauncher.
// Test TestLauncher filters, and command line switches setup.
class TestLauncherTest : public testing::Test { … };

class ResultWatcherTest : public testing::Test { … };

// Action to mock delegate invoking OnTestFinish on test launcher.
ACTION_P3(OnTestResult, launcher, full_name, status) { … }

// Action to mock delegate invoking OnTestFinish on test launcher.
ACTION_P2(OnTestResult, launcher, result) { … }

// A test and a disabled test cannot share a name.
TEST_F(TestLauncherTest, TestNameSharedWithDisabledTest) { … }

// A test case and a disabled test case cannot share a name.
TEST_F(TestLauncherTest, TestNameSharedWithDisabledTestCase) { … }

// Compiled tests should not contain an orphaned pre test.
TEST_F(TestLauncherTest, OrphanePreTest) { … }

// When There are no tests, delegate should not be called.
TEST_F(TestLauncherTest, EmptyTestSetPasses) { … }

// Test TestLauncher filters DISABLED tests by default.
TEST_F(TestLauncherTest, FilterDisabledTestByDefault) { … }

// Test TestLauncher should reorder PRE_ tests before delegate
TEST_F(TestLauncherTest, ReorderPreTests) { … }

// Test TestLauncher "gtest_filter" switch.
TEST_F(TestLauncherTest, UsingCommandLineFilter) { … }

// Test TestLauncher gtest filter will include pre tests
TEST_F(TestLauncherTest, FilterIncludePreTest) { … }

// Test TestLauncher gtest filter works when both include and exclude filter
// are defined.
TEST_F(TestLauncherTest, FilterIncludeExclude) { … }

// Test TestLauncher "gtest_repeat" switch.
TEST_F(TestLauncherTest, RepeatTest) { … }

// Test TestLauncher --gtest_repeat and --gtest_break_on_failure.
TEST_F(TestLauncherTest, RunningMultipleIterationsUntilFailure) { … }

// Test TestLauncher will retry failed test, and stop on success.
TEST_F(TestLauncherTest, SuccessOnRetryTests) { … }

// Test TestLauncher will retry continuing failing test up to retry limit,
// before eventually failing and returning false.
TEST_F(TestLauncherTest, FailOnRetryTests) { … }

// Test TestLauncher should retry all PRE_ chained tests
TEST_F(TestLauncherTest, RetryPreTests) { … }

// Test TestLauncher should fail if a PRE test fails but its non-PRE test passes
TEST_F(TestLauncherTest, PreTestFailure) { … }

// Test TestLauncher run disabled unit tests switch.
TEST_F(TestLauncherTest, RunDisabledTests) { … }

// Test TestLauncher does not run negative tests filtered under
// testing/buildbot/filters.
TEST_F(TestLauncherTest, DoesRunFilteredTests) { … }

// Test TestLauncher run disabled tests and negative tests filtered under
// testing/buildbot/filters, when gtest_also_run_disabled_tests is set.
TEST_F(TestLauncherTest, RunDisabledTestsWithFilteredTests) { … }

// Disabled test should disable all pre tests
TEST_F(TestLauncherTest, DisablePreTests) { … }

// Test TestLauncher enforce to run tests in the exact positive filter.
TEST_F(TestLauncherTest, EnforceRunTestsInExactPositiveFilter) { … }

// Test TestLauncher should fail if enforce-exact-positive-filter and
// gtest_filter both presented.
TEST_F(TestLauncherTest,
       EnforceRunTestsInExactPositiveFailWithGtestFilterFlag) { … }

// Test TestLauncher should fail if enforce-exact-positive-filter is set
// with negative test filters.
TEST_F(TestLauncherTest, EnforceRunTestsInExactPositiveFailWithNegativeFilter) { … }

// Test TestLauncher should fail if enforce-exact-positive-filter is set
// with wildcard positive filters.
TEST_F(TestLauncherTest,
       EnforceRunTestsInExactPositiveFailWithWildcardPositiveFilter) { … }

// Tests fail if they produce too much output.
TEST_F(TestLauncherTest, ExcessiveOutput) { … }

// Use command-line switch to allow more output.
TEST_F(TestLauncherTest, OutputLimitSwitch) { … }

// Shard index must be lesser than total shards
TEST_F(TestLauncherTest, FaultyShardSetup) { … }

// Shard index must be lesser than total shards
TEST_F(TestLauncherTest, RedirectStdio) { … }

// Sharding should be stable and always selecting the same tests.
TEST_F(TestLauncherTest, StableSharding) { … }

// Validate |iteration_data| contains one test result matching |result|.
bool ValidateTestResultObject(const Value::Dict& iteration_data,
                              TestResult& test_result) { … }

// Validate |root| dictionary value contains a list with |values|
// at |key| value.
bool ValidateStringList(const std::optional<Value::Dict>& root,
                        const std::string& key,
                        std::vector<const char*> values) { … }

// Unit tests to validate TestLauncher outputs the correct JSON file.
TEST_F(TestLauncherTest, JsonSummary) { … }

// Validate TestLauncher outputs the correct JSON file
// when running disabled tests.
TEST_F(TestLauncherTest, JsonSummaryWithDisabledTests) { … }

// Matches a std::tuple<const FilePath&, const FilePath&> where the first
// item is a parent of the second.
MATCHER(DirectoryIsParentOf, "") { … }

// Test that the launcher creates a dedicated temp dir for a child proc and
// cleans it up.
TEST_F(TestLauncherTest, TestChildTempDir) { … }

#if BUILDFLAG(IS_FUCHSIA)
// Verifies that test processes have /data, /cache and /tmp available.
TEST_F(TestLauncherTest, ProvidesDataCacheAndTmpDirs) {
  EXPECT_TRUE(base::DirectoryExists(base::FilePath("/data")));
  EXPECT_TRUE(base::DirectoryExists(base::FilePath("/cache")));
  EXPECT_TRUE(base::DirectoryExists(base::FilePath("/tmp")));
}
#endif  // BUILDFLAG(IS_FUCHSIA)

// Unit tests to validate UnitTestLauncherDelegate implementation.
class UnitTestLauncherDelegateTester : public testing::Test { … };

// Validate delegate produces correct command line.
TEST_F(UnitTestLauncherDelegateTester, GetCommandLine) { … }

// Verify that a result watcher can stop polling early when all tests complete.
TEST_F(ResultWatcherTest, PollCompletesQuickly) { … }

// Verify that a result watcher repeatedly checks the file for a batch of slow
// tests. Each test completes in 40s, which is just under the timeout of 45s.
TEST_F(ResultWatcherTest, PollCompletesSlowly) { … }

// Verify that the result watcher identifies when a test times out.
TEST_F(ResultWatcherTest, PollTimeout) { … }

// Verify that the result watcher retries incomplete reads.
TEST_F(ResultWatcherTest, RetryIncompleteResultRead) { … }

// Verify that the result watcher continues polling with the base timeout when
// the clock jumps backward.
TEST_F(ResultWatcherTest, PollWithClockJumpBackward) { … }

// Verify that the result watcher continues polling with the base timeout when
// the clock jumps forward.
TEST_F(ResultWatcherTest, PollWithClockJumpForward) { … }

// Validate delegate sets batch size correctly.
TEST_F(UnitTestLauncherDelegateTester, BatchSize) { … }

// The following 4 tests are disabled as they are meant to only run from
// |RunMockTests| to validate tests launcher output for known results. The tests
// are expected to run in order within a same batch.

// Basic test to pass
TEST(MockUnitTests, DISABLED_PassTest) { … }
// Basic test to fail
TEST(MockUnitTests, DISABLED_FailTest) { … }
// Basic test to crash
TEST(MockUnitTests, DISABLED_CrashTest) { … }
// Basic test will not be reached, due to the preceding crash in the same batch.
TEST(MockUnitTests, DISABLED_NoRunTest) { … }

// Using TestLauncher to launch 3 basic unitests
// and validate the resulting json file.
TEST_F(UnitTestLauncherDelegateTester, RunMockTests) { … }

TEST(ProcessGTestOutputTest, RunMockTests) { … }

// TODO(crbug.com/40287376): Enable the test once GetAppOutputAndError
// can collect stdout and stderr on Fuchsia.
#if !BUILDFLAG(IS_FUCHSIA)
TEST(ProcessGTestOutputTest, FoundTestCaseNotEnforced) { … }
#endif  // !BUILDFLAG(IS_FUCHSIA)

// TODO(crbug.com/40135391): Enable leaked-child checks on other platforms.
#if BUILDFLAG(IS_FUCHSIA)

// Test that leaves a child process running. The test is DISABLED_, so it can
// be launched explicitly by RunMockLeakProcessTest

MULTIPROCESS_TEST_MAIN(LeakChildProcess) {
  while (true)
    PlatformThread::Sleep(base::Seconds(1));
}

TEST(LeakedChildProcessTest, DISABLED_LeakChildProcess) {
  Process child_process = SpawnMultiProcessTestChild(
      "LeakChildProcess", GetMultiProcessTestChildBaseCommandLine(),
      LaunchOptions());
  ASSERT_TRUE(child_process.IsValid());
  // Don't wait for the child process to exit.
}

// Validate that a test that leaks a process causes the batch to have an
// error exit_code.
TEST_F(UnitTestLauncherDelegateTester, LeakedChildProcess) {
  CommandLine command_line(CommandLine::ForCurrentProcess()->GetProgram());
  command_line.AppendSwitchASCII(
      "gtest_filter", "LeakedChildProcessTest.DISABLED_LeakChildProcess");

  ASSERT_TRUE(dir.CreateUniqueTempDir());
  FilePath path = dir.GetPath().AppendASCII("SaveSummaryResult.json");
  command_line.AppendSwitchPath("test-launcher-summary-output", path);
  command_line.AppendSwitch("gtest_also_run_disabled_tests");
  command_line.AppendSwitchASCII("test-launcher-retry-limit", "0");

  std::string output;
  int exit_code = 0;
  GetAppOutputWithExitCode(command_line, &output, &exit_code);

  // Validate that we actually ran a test.
  std::optional<Value::Dict> root = test_launcher_utils::ReadSummary(path);
  ASSERT_TRUE(root);

  Value::Dict* dict = root->FindDict("test_locations");
  ASSERT_TRUE(dict);
  EXPECT_EQ(1u, dict->size());

  EXPECT_TRUE(test_launcher_utils::ValidateTestLocations(
      *dict, "LeakedChildProcessTest"));

  // Validate that the leaked child caused the batch to error-out.
  EXPECT_EQ(exit_code, 1);
}
#endif  // BUILDFLAG(IS_FUCHSIA)

// Validate GetTestOutputSnippetTest assigns correct output snippet.
TEST(TestLauncherTools, GetTestOutputSnippetTest) { … }

MATCHER(CheckTruncationPreservesMessage, "") { … }

void MatchesFatalMessagesTest() { … }

// Validates TestSnippetFocused correctly identifies fatal messages to
// retain during truncation.
TEST(TestLauncherTools, TruncateSnippetFocusedMatchesFatalMessagesTest) { … }

#if BUILDFLAG(IS_CHROMEOS_ASH)
// Validates TestSnippetFocused correctly identifies fatal messages to
// retain during truncation, for ChromeOS Ash.
TEST(TestLauncherTools, TruncateSnippetFocusedMatchesFatalMessagesCrosAshTest) {
  logging::ScopedLoggingSettings scoped_logging_settings;
  scoped_logging_settings.SetLogFormat(logging::LogFormat::LOG_FORMAT_CHROME);
  MatchesFatalMessagesTest();
}
#endif

// Validate TestSnippetFocused truncates snippets correctly, regardless of
// whether fatal messages appear at the start, middle or end of the snippet.
TEST(TestLauncherTools, TruncateSnippetFocusedTest) { … }

}  // namespace

}  // namespace base
chromium/base/test/launcher/test_launcher_unittest.cc
