TestSuite.cpp | Explore in Territory

//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite:
//   Basic implementation of a test harness in ANGLE.

#include "TestSuite.h"

#include "common/debug.h"
#include "common/hash_containers.h"
#include "common/platform.h"
#include "common/string_utils.h"
#include "common/system_utils.h"
#include "util/Timer.h"

#include <stdlib.h>
#include <time.h>

#include <fstream>
#include <unordered_map>

#include <gtest/gtest.h>
#include <rapidjson/document.h>
#include <rapidjson/filewritestream.h>
#include <rapidjson/istreamwrapper.h>
#include <rapidjson/prettywriter.h>

// We directly call into a function to register the parameterized tests. This saves spinning up
// a subprocess with a new gtest filter.
#include <gtest/../../src/gtest-internal-inl.h>

js;

namespace angle
{
namespace
{
constexpr char kBatchId[]             = …;
constexpr char kFilterFileArg[]       = …;
constexpr char kResultFileArg[]       = …;
constexpr char kTestTimeoutArg[]      = …;
constexpr char kDisableCrashHandler[] = …;
constexpr char kIsolatedOutDir[]      = …;

constexpr char kStartedTestString[] = …;
constexpr char kPassedTestString[]  = …;
constexpr char kFailedTestString[]  = …;
constexpr char kSkippedTestString[] = …;

constexpr char kArtifactsFakeTestName[] = …;

constexpr char kTSanOptionsEnvVar[]  = …;
constexpr char kUBSanOptionsEnvVar[] = …;

[[maybe_unused]] constexpr char kVkLoaderDisableDLLUnloadingEnvVar[] = …;

// Note: we use a fairly high test timeout to allow for the first test in a batch to be slow.
// Ideally we could use a separate timeout for the slow first test.
// Allow sanitized tests to run more slowly.
#if defined(NDEBUG) && !defined(ANGLE_WITH_SANITIZER)
constexpr int kDefaultTestTimeout  = 60;
constexpr int kDefaultBatchTimeout = 300;
#else
constexpr int kDefaultTestTimeout  = …;
constexpr int kDefaultBatchTimeout = …;
#endif
constexpr int kSlowTestTimeoutScale  = …;
constexpr int kDefaultBatchSize      = …;
constexpr double kIdleMessageTimeout = …;
constexpr int kDefaultMaxProcesses   = …;
constexpr int kDefaultMaxFailures    = …;

const char *ResultTypeToString(TestResultType type)
{ … }

TestResultType GetResultTypeFromString(const std::string &str)
{ … }

bool IsFailedResult(TestResultType resultType)
{ … }

js::Value ResultTypeToJSString(TestResultType type, js::Document::AllocatorType *allocator)
{ … }

bool WriteJsonFile(const std::string &outputFile, js::Document *doc)
{ … }

// Writes out a TestResults to the Chromium JSON Test Results format.
// https://chromium.googlesource.com/chromium/src.git/+/main/docs/testing/json_test_results_format.md
void WriteResultsFile(bool interrupted,
                      const TestResults &testResults,
                      const std::string &outputFile)
{ … }

void WriteHistogramJson(const HistogramWriter &histogramWriter, const std::string &outputFile)
{ … }

void UpdateCurrentTestResult(const testing::TestResult &resultIn, TestResults *resultsOut)
{ … }

TestIdentifier GetTestIdentifier(const testing::TestInfo &testInfo)
{ … }

bool IsTestDisabled(const testing::TestInfo &testInfo)
{ … }

TestIdentifierFilter;

std::vector<TestIdentifier> FilterTests(std::map<TestIdentifier, FileLine> *fileLinesOut,
                                        TestIdentifierFilter filter,
                                        bool alsoRunDisabledTests)
{ … }

std::vector<TestIdentifier> GetFilteredTests(std::map<TestIdentifier, FileLine> *fileLinesOut,
                                             bool alsoRunDisabledTests)
{ … }

std::vector<TestIdentifier> GetShardTests(const std::vector<TestIdentifier> &allTests,
                                          int shardIndex,
                                          int shardCount,
                                          std::map<TestIdentifier, FileLine> *fileLinesOut,
                                          bool alsoRunDisabledTests)
{ … }

std::string GetTestFilter(const std::vector<TestIdentifier> &tests)
{ … }

bool GetTestArtifactsFromJSON(const js::Value::ConstObject &obj,
                              std::vector<std::string> *testArtifactPathsOut)
{ … }

bool GetSingleTestResultFromJSON(const js::Value &name,
                                 const js::Value::ConstObject &obj,
                                 TestResults *resultsOut)
{ … }

bool GetTestResultsFromJSON(const js::Document &document, TestResults *resultsOut)
{ … }

bool MergeTestResults(TestResults *input, TestResults *output, int flakyRetries)
{ … }

void PrintTestOutputSnippet(const TestIdentifier &id,
                            const TestResult &result,
                            const std::string &fullOutput)
{ … }

std::string GetConfigNameFromTestIdentifier(const TestIdentifier &id)
{ … }

TestQueue BatchTests(const std::vector<TestIdentifier> &tests, int batchSize)
{ … }

void ListTests(const std::map<TestIdentifier, TestResult> &resultsMap)
{ … }

// Prints the names of the tests matching the user-specified filter flag.
// This matches the output from googletest/src/gtest.cc but is much much faster for large filters.
// See http://anglebug.com/42263725
void GTestListTests(const std::map<TestIdentifier, TestResult> &resultsMap)
{ … }

// On Android, batching is done on the host, i.e. externally.
// TestSuite executes on the device and should just passthrough all args to GTest.
bool UsesExternalBatching()
{ … }
}  // namespace

void MetricWriter::enable(const std::string &testArtifactDirectory)
{ … }

void MetricWriter::writeInfo(const std::string &name,
                             const std::string &backend,
                             const std::string &story,
                             const std::string &metric,
                             const std::string &units)
{ … }

void MetricWriter::writeDoubleValue(double value)
{ … }

void MetricWriter::writeIntegerValue(size_t value)
{ … }

void MetricWriter::close()
{ … }

// static
TestSuite *TestSuite::mInstance = …;

TestIdentifier::TestIdentifier() = default;

TestIdentifier::TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn)
    : … { … }

TestIdentifier::TestIdentifier(const TestIdentifier &other) = default;

TestIdentifier::~TestIdentifier() = default;

TestIdentifier &TestIdentifier::operator=(const TestIdentifier &other) = default;

void TestIdentifier::snprintfName(char *outBuffer, size_t maxLen) const
{ … }

// static
bool TestIdentifier::ParseFromString(const std::string &str, TestIdentifier *idOut)
{ … }

TestResults::TestResults() = default;

TestResults::~TestResults() = default;

ProcessInfo::ProcessInfo() = default;

ProcessInfo &ProcessInfo::operator=(ProcessInfo &&rhs)
{ … }

ProcessInfo::~ProcessInfo() = default;

ProcessInfo::ProcessInfo(ProcessInfo &&other)
{ … }

class TestSuite::TestEventListener : public testing::EmptyTestEventListener
{ … };

TestSuite::TestSuite(int *argc, char **argv) : … { … }

TestSuite::TestSuite(int *argc, char **argv, std::function<void()> registerTestsCallback)
    : … { … }

TestSuite::~TestSuite()
{ … }

bool TestSuite::parseSingleArg(int *argc, char **argv, int argIndex)
{ … }

void TestSuite::onCrashOrTimeout(TestResultType crashOrTimeout)
{ … }

bool TestSuite::launchChildTestProcess(uint32_t batchId,
                                       const std::vector<TestIdentifier> &testsInBatch)
{ … }

void ParseTestIdentifierAndSetResult(const std::string &testName,
                                     TestResultType result,
                                     TestResults *results)
{ … }

bool TestSuite::finishProcess(ProcessInfo *processInfo)
{ … }

int TestSuite::run()
{ … }

int TestSuite::printFailuresAndReturnCount() const
{ … }

void TestSuite::startWatchdog()
{ … }

void TestSuite::addHistogramSample(const std::string &measurement,
                                   const std::string &story,
                                   double value,
                                   const std::string &units)
{ … }

bool TestSuite::hasTestArtifactsDirectory() const
{ … }

std::string TestSuite::reserveTestArtifactPath(const std::string &artifactName)
{ … }

bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut)
{ … }

void TestSuite::dumpTestExpectationsErrorMessages()
{ … }

bool TestSuite::loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config,
                                                       const std::string &fileName)
{ … }

bool TestSuite::loadAllTestExpectationsFromFile(const std::string &fileName)
{ … }

bool TestSuite::logAnyUnusedTestExpectations()
{ … }

int32_t TestSuite::getTestExpectation(const std::string &testName)
{ … }

void TestSuite::maybeUpdateTestTimeout(uint32_t testExpectation)
{ … }

int32_t TestSuite::getTestExpectationWithConfigAndUpdateTimeout(const GPUTestConfig &config,
                                                                const std::string &testName)
{ … }

int TestSuite::getSlowTestTimeout() const
{ … }

void TestSuite::writeOutputFiles(bool interrupted)
{ … }

const char *TestResultTypeToString(TestResultType type)
{ … }

// This code supports using "-" in test names, which happens often in dEQP. GTest uses as a marker
// for the beginning of the exclusion filter. Work around this by replacing "-" with "?" which
// matches any single character.
std::string ReplaceDashesWithQuestionMark(std::string dashesString)
{ … }
}  // namespace angle
chromium/third_party/angle/src/tests/test_utils/runner/TestSuite.cpp
