chromium/testing/perf/confidence/compare_blink_perf.cc

// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// A utility to compare two runs of a style perftest, in the perftest
// output format, and compute confidence intervals. The simplest way
// to get the right information is to build a binary before and after
// changes and then run it continuously every-other for a while until
// you feel you have enough data:
//
//   rm -f old.txt new.txt; \
//   while :; do
//     taskset -c 2,4,6,8 ./out/Release/blink_perf___old \
//       --gtest_filter=StyleCalcPerfTest.\* 2>&1 | tee -a old.txt;
//     taskset -c 2,4,6,8 ./out/Release/blink_perf_tests \
//       --gtest_filter=StyleCalcPerfTest.\* 2>&1 | tee -a new.txt;
//   done
//
// and then run ./out/Release/compare_blink_perf old.txt new.txt.
// (Possibly under watch -n 1 if you want to look at data as it
// comes in, though beware of p-hacking.)
//
// TODO(sesse): Consider whether we should remove the first few
// runs, as they are frequently outliers.

#include <stdio.h>
#include <stdlib.h>

#include <algorithm>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

#include "base/rand_util.h"
#include "base/strings/string_number_conversions.h"
#include "testing/perf/confidence/ratio_bootstrap_estimator.h"

#ifdef UNSAFE_BUFFERS_BUILD
// Not used with untrusted inputs.
#pragma allow_unsafe_buffers
#endif

max;
min;
numeric_limits;
pair;
sort;
string;
unordered_map;
vector;

namespace {

string BeautifyCategory(const string& category) { … }

bool CodeUnitCompareIgnoringASCIICaseLessThan(const string& a,
                                              const string& b) { … }

// The structure is e.g. BlinkStyleParseTime -> Video -> [100 us, 90 us, ...]
unordered_map<string, unordered_map<string, vector<double>>> ReadFile(
    const char* filename) { … }

// Find the number of trials, for display.
void FindNumberOfTrials(
    const unordered_map<string, unordered_map<string, vector<double>>>&
        measurements,
    unsigned& min_num_trials,
    unsigned& max_num_trials) { … }

struct Label { … };

}  // namespace

int main(int argc, char** argv) { … }