uniformity_unittest.cc | Explore in Territory

// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <stdint.h>

#include <initializer_list>
#include <map>
#include <memory>
#include <optional>

#include "base/feature_list.h"
#include "base/functional/bind.h"
#include "base/strings/stringprintf.h"
#include "base/test/scoped_feature_list.h"
#include "components/variations/entropy_provider.h"
#include "components/variations/proto/study.pb.h"
#include "components/variations/proto/variations_seed.pb.h"
#include "components/variations/variations_layers.h"
#include "components/variations/variations_seed_processor.h"
#include "components/variations/variations_test_utils.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"

namespace variations {
namespace {

// For these tests, we use a small LES range to make the expectations simpler.
const uint32_t kMaxEntropy = …;
const uint32_t kLayerId = …;
const uint32_t kLayerSalt = …;
const char kStudyName[] = …;

struct LayerStudySeedOptions { … };

// Generates a seed for checking uniformity of assignments in layered
// constrained study. This seed contains a 3 arm study active in 9/10 slots.
// |slot_multiplier| increased the number of slots in each range, but should not
// affect randomization.
VariationsSeed LayerStudySeed(const LayerStudySeedOptions& options) { … }

// A vector of 20 empty strings representing 20 clients each with an empty
// assignment.
const std::vector<std::string> kNoAssignments(20, "");

// When assigned directly from low entropy, the following assignments are used
// for the study.
const std::vector<std::string> kExpectedLowEntropyAssignments = …;

// LayeredStudySeed should give the following assignment using the test LES.
// All 3 arms get 6/20 values, with 2/20 not in the study.
const std::vector<std::string> kExpectedRemainderEntropyAssignments = …;

// The assignment results using the limited entropy provider. The setup in
// LayerStudySeed() implies 10% of the client will not have an active layer
// member, and thus will not receive an assignment. In the simulation, 1/20
// comes out to be empty (the one at index 11), which is a likely event (p=0.27)
// given the setup.
const std::vector<std::string> kExpectedLimitedEntropyAssignments = …;

// The expected group assignments for the study based on high entropy.
// This does not take into account any layer exclusions.
// This is only a small sample of the entropy space, so we don't expect
// precised uniformity, just a reasonable mixture.
const std::vector<std::string> kExpectedHighEntropyStudyAssignments = …;

// Process the seed and return which group the user is assigned for Uniformity.
// From the setup in LayerStudySeed(), the group (i.e., the return value) can be
// either "A", "B", "C", or an empty string when the study is not assigned.
std::string GetUniformityAssignment(const VariationsSeed& seed,
                                    const EntropyProviders& entropy_providers) { … }

// Processes the seed and returns which group the user is assigned. It uses a
// total of 120 simulated clients and returns the 120 assignment results as a
// vector. See GetUniformityAssignment() for an explanation on each of the
// result. The first 20 simulated clients do not have client IDs, and each has
// a distinct low entropy value. The next 100 clients all have client IDs, and
// are uniformly assigned to one of the 20 low entropy values (i.e., 5 clients
// per value).
std::vector<std::string> GetUniformityAssignments(
    const VariationsSeed& seed,
    bool enable_benchmarking = false) { … }

// Performs randomization from the given seed to 40 simulated clients and
// returns the group assignment for each client. The group assignment can be
// "A", "B", "C", or empty string if the client is not assigned. The first 20
// clients do not have a limited entropy randomization source. The last 20
// clients each has a different non-empty limited entropy randomization source.
std::vector<std::string> GetUniformityAssignmentsWithVaryingLimitedSource(
    const VariationsSeed& seed,
    bool enable_benchmarking = false) { … }

std::vector<std::string> Concat(
    std::initializer_list<const std::vector<std::string>*> vectors) { … }

// Computes the Chi-Square statistic for |assignment_counts| assuming they
// follow a uniform distribution, where each entry has expected value
// |expected_value|.
//
// The Chi-Square statistic is defined as Sum((O-E)^2/E) where O is the observed
// value and E is the expected value.
double ComputeChiSquare(const std::map<std::string, size_t>& assignment_counts,
                        double expected_value) { … }

}  // namespace

// We should get the same assignments for clients that have no high entropy.
// This should be true for both types of layer entropy.
TEST(VariationsUniformityTest, UnlayeredDefaultEntropyStudy) { … }

// We should get the same assignments for clients that have no high entropy.
// This should be true for both types of layer entropy.
TEST(VariationsUniformityTest, UnlayeredLowEntropyStudy) { … }

TEST(VariationsUniformityTest, LowEntropyLayerDefaultEntropyStudy) { … }

TEST(VariationsUniformityTest, LowEntropyLayerLowEntropyStudy) { … }

// We should get the same assignments for clients that have no high entropy.
// This should be true for both types of layer entropy.
TEST(VariationsUniformityTest, DefaultEntropyLayerDefaultEntropyStudy) { … }

TEST(VariationsUniformityTest, LimitedEntropyLayerLimitedEntropyStudy) { … }

TEST(VariationsUniformityTest, LimitedEntropyLayerDefaultEntropyStudy) { … }

// Not specifying the `salt` field for the layer should fall back to using the
// `id` field as salt. Different salt values should result in different layer
// exclusions.
TEST(VariationsUniformityTest, LayerSalt) { … }

// When enable_benchmarking is passed, layered studies should never activate.
TEST(VariationsUniformityTest, BenchmarkingDisablesLayeredStudies) { … }

// When enable_benchmarking is passed, layered studies should never activate.
TEST(VariationsUniformityTest,
     BenchmarkingDisablesLayeredStudies_LimitedEntropy) { … }

TEST(VariationsUniformityTest, SessionEntropyStudyChiSquare) { … }

}  // namespace variations
chromium/components/variations/uniformity_unittest.cc
