#!/usr/bin/env python3
# type: ignore
"""
compare.py - versatile benchmark output compare tool
"""
import argparse
import json
import os
import sys
import unittest
from argparse import ArgumentParser
import gbench
from gbench import report, util
def check_inputs(in1, in2, flags):
"""
Perform checking on the user provided inputs and diagnose any abnormalities
"""
in1_kind, in1_err = util.classify_input_file(in1)
in2_kind, in2_err = util.classify_input_file(in2)
output_file = util.find_benchmark_flag("--benchmark_out=", flags)
output_type = util.find_benchmark_flag("--benchmark_out_format=", flags)
if (
in1_kind == util.IT_Executable
and in2_kind == util.IT_Executable
and output_file
):
print(
(
"WARNING: '--benchmark_out=%s' will be passed to both "
"benchmarks causing it to be overwritten"
)
% output_file
)
if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:
# When both sides are JSON the only supported flag is
# --benchmark_filter=
for flag in util.remove_benchmark_flags("--benchmark_filter=", flags):
print(
"WARNING: passing %s has no effect since both "
"inputs are JSON" % flag
)
if output_type is not None and output_type != "json":
print(
(
"ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
" is not supported."
)
% output_type
)
sys.exit(1)
def create_parser():
parser = ArgumentParser(
description="versatile benchmark output compare tool"
)
parser.add_argument(
"-a",
"--display_aggregates_only",
dest="display_aggregates_only",
action="store_true",
help="If there are repetitions, by default, we display everything - the"
" actual runs, and the aggregates computed. Sometimes, it is "
"desirable to only view the aggregates. E.g. when there are a lot "
"of repetitions. Do note that only the display is affected. "
"Internally, all the actual runs are still used, e.g. for U test.",
)
parser.add_argument(
"--no-color",
dest="color",
default=True,
action="store_false",
help="Do not use colors in the terminal output",
)
parser.add_argument(
"-d",
"--dump_to_json",
dest="dump_to_json",
help="Additionally, dump benchmark comparison output to this file in JSON format.",
)
utest = parser.add_argument_group()
utest.add_argument(
"--no-utest",
dest="utest",
default=True,
action="store_false",
help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(
report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS
),
)
alpha_default = 0.05
utest.add_argument(
"--alpha",
dest="utest_alpha",
default=alpha_default,
type=float,
help=(
"significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)"
)
% alpha_default,
)
subparsers = parser.add_subparsers(
help="This tool has multiple modes of operation:", dest="mode"
)
parser_a = subparsers.add_parser(
"benchmarks",
help="The most simple use-case, compare all the output of these two benchmarks",
)
baseline = parser_a.add_argument_group("baseline", "The benchmark baseline")
baseline.add_argument(
"test_baseline",
metavar="test_baseline",
type=argparse.FileType("r"),
nargs=1,
help="A benchmark executable or JSON output file",
)
contender = parser_a.add_argument_group(
"contender", "The benchmark that will be compared against the baseline"
)
contender.add_argument(
"test_contender",
metavar="test_contender",
type=argparse.FileType("r"),
nargs=1,
help="A benchmark executable or JSON output file",
)
parser_a.add_argument(
"benchmark_options",
metavar="benchmark_options",
nargs=argparse.REMAINDER,
help="Arguments to pass when running benchmark executables",
)
parser_b = subparsers.add_parser(
"filters", help="Compare filter one with the filter two of benchmark"
)
baseline = parser_b.add_argument_group("baseline", "The benchmark baseline")
baseline.add_argument(
"test",
metavar="test",
type=argparse.FileType("r"),
nargs=1,
help="A benchmark executable or JSON output file",
)
baseline.add_argument(
"filter_baseline",
metavar="filter_baseline",
type=str,
nargs=1,
help="The first filter, that will be used as baseline",
)
contender = parser_b.add_argument_group(
"contender", "The benchmark that will be compared against the baseline"
)
contender.add_argument(
"filter_contender",
metavar="filter_contender",
type=str,
nargs=1,
help="The second filter, that will be compared against the baseline",
)
parser_b.add_argument(
"benchmark_options",
metavar="benchmark_options",
nargs=argparse.REMAINDER,
help="Arguments to pass when running benchmark executables",
)
parser_c = subparsers.add_parser(
"benchmarksfiltered",
help="Compare filter one of first benchmark with filter two of the second benchmark",
)
baseline = parser_c.add_argument_group("baseline", "The benchmark baseline")
baseline.add_argument(
"test_baseline",
metavar="test_baseline",
type=argparse.FileType("r"),
nargs=1,
help="A benchmark executable or JSON output file",
)
baseline.add_argument(
"filter_baseline",
metavar="filter_baseline",
type=str,
nargs=1,
help="The first filter, that will be used as baseline",
)
contender = parser_c.add_argument_group(
"contender", "The benchmark that will be compared against the baseline"
)
contender.add_argument(
"test_contender",
metavar="test_contender",
type=argparse.FileType("r"),
nargs=1,
help="The second benchmark executable or JSON output file, that will be compared against the baseline",
)
contender.add_argument(
"filter_contender",
metavar="filter_contender",
type=str,
nargs=1,
help="The second filter, that will be compared against the baseline",
)
parser_c.add_argument(
"benchmark_options",
metavar="benchmark_options",
nargs=argparse.REMAINDER,
help="Arguments to pass when running benchmark executables",
)
return parser
def main():
# Parse the command line flags
parser = create_parser()
args, unknown_args = parser.parse_known_args()
if args.mode is None:
parser.print_help()
exit(1)
assert not unknown_args
benchmark_options = args.benchmark_options
if args.mode == "benchmarks":
test_baseline = args.test_baseline[0].name
test_contender = args.test_contender[0].name
filter_baseline = ""
filter_contender = ""
# NOTE: if test_baseline == test_contender, you are analyzing the stdev
description = "Comparing %s to %s" % (test_baseline, test_contender)
elif args.mode == "filters":
test_baseline = args.test[0].name
test_contender = args.test[0].name
filter_baseline = args.filter_baseline[0]
filter_contender = args.filter_contender[0]
# NOTE: if filter_baseline == filter_contender, you are analyzing the
# stdev
description = "Comparing %s to %s (from %s)" % (
filter_baseline,
filter_contender,
args.test[0].name,
)
elif args.mode == "benchmarksfiltered":
test_baseline = args.test_baseline[0].name
test_contender = args.test_contender[0].name
filter_baseline = args.filter_baseline[0]
filter_contender = args.filter_contender[0]
# NOTE: if test_baseline == test_contender and
# filter_baseline == filter_contender, you are analyzing the stdev
description = "Comparing %s (from %s) to %s (from %s)" % (
filter_baseline,
test_baseline,
filter_contender,
test_contender,
)
else:
# should never happen
print("Unrecognized mode of operation: '%s'" % args.mode)
parser.print_help()
exit(1)
check_inputs(test_baseline, test_contender, benchmark_options)
if args.display_aggregates_only:
benchmark_options += ["--benchmark_display_aggregates_only=true"]
options_baseline = []
options_contender = []
if filter_baseline and filter_contender:
options_baseline = ["--benchmark_filter=%s" % filter_baseline]
options_contender = ["--benchmark_filter=%s" % filter_contender]
# Run the benchmarks and report the results
json1 = json1_orig = gbench.util.sort_benchmark_results(
gbench.util.run_or_load_benchmark(
test_baseline, benchmark_options + options_baseline
)
)
json2 = json2_orig = gbench.util.sort_benchmark_results(
gbench.util.run_or_load_benchmark(
test_contender, benchmark_options + options_contender
)
)
# Now, filter the benchmarks so that the difference report can work
if filter_baseline and filter_contender:
replacement = "[%s vs. %s]" % (filter_baseline, filter_contender)
json1 = gbench.report.filter_benchmark(
json1_orig, filter_baseline, replacement
)
json2 = gbench.report.filter_benchmark(
json2_orig, filter_contender, replacement
)
diff_report = gbench.report.get_difference_report(json1, json2, args.utest)
output_lines = gbench.report.print_difference_report(
diff_report,
args.display_aggregates_only,
args.utest,
args.utest_alpha,
args.color,
)
print(description)
for ln in output_lines:
print(ln)
# Optionally, diff and output to JSON
if args.dump_to_json is not None:
with open(args.dump_to_json, "w") as f_json:
json.dump(diff_report, f_json, indent=1)
class TestParser(unittest.TestCase):
def setUp(self):
self.parser = create_parser()
testInputs = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs"
)
self.testInput0 = os.path.join(testInputs, "test1_run1.json")
self.testInput1 = os.path.join(testInputs, "test1_run2.json")
def test_benchmarks_basic(self):
parsed = self.parser.parse_args(
["benchmarks", self.testInput0, self.testInput1]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "benchmarks")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertFalse(parsed.benchmark_options)
def test_benchmarks_basic_without_utest(self):
parsed = self.parser.parse_args(
["--no-utest", "benchmarks", self.testInput0, self.testInput1]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertFalse(parsed.utest)
self.assertEqual(parsed.utest_alpha, 0.05)
self.assertEqual(parsed.mode, "benchmarks")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertFalse(parsed.benchmark_options)
def test_benchmarks_basic_display_aggregates_only(self):
parsed = self.parser.parse_args(
["-a", "benchmarks", self.testInput0, self.testInput1]
)
self.assertTrue(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "benchmarks")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertFalse(parsed.benchmark_options)
def test_benchmarks_basic_with_utest_alpha(self):
parsed = self.parser.parse_args(
["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.utest_alpha, 0.314)
self.assertEqual(parsed.mode, "benchmarks")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertFalse(parsed.benchmark_options)
def test_benchmarks_basic_without_utest_with_utest_alpha(self):
parsed = self.parser.parse_args(
[
"--no-utest",
"--alpha=0.314",
"benchmarks",
self.testInput0,
self.testInput1,
]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertFalse(parsed.utest)
self.assertEqual(parsed.utest_alpha, 0.314)
self.assertEqual(parsed.mode, "benchmarks")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertFalse(parsed.benchmark_options)
def test_benchmarks_with_remainder(self):
parsed = self.parser.parse_args(
["benchmarks", self.testInput0, self.testInput1, "d"]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "benchmarks")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertEqual(parsed.benchmark_options, ["d"])
def test_benchmarks_with_remainder_after_doubleminus(self):
parsed = self.parser.parse_args(
["benchmarks", self.testInput0, self.testInput1, "--", "e"]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "benchmarks")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertEqual(parsed.benchmark_options, ["e"])
def test_filters_basic(self):
parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"])
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "filters")
self.assertEqual(parsed.test[0].name, self.testInput0)
self.assertEqual(parsed.filter_baseline[0], "c")
self.assertEqual(parsed.filter_contender[0], "d")
self.assertFalse(parsed.benchmark_options)
def test_filters_with_remainder(self):
parsed = self.parser.parse_args(
["filters", self.testInput0, "c", "d", "e"]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "filters")
self.assertEqual(parsed.test[0].name, self.testInput0)
self.assertEqual(parsed.filter_baseline[0], "c")
self.assertEqual(parsed.filter_contender[0], "d")
self.assertEqual(parsed.benchmark_options, ["e"])
def test_filters_with_remainder_after_doubleminus(self):
parsed = self.parser.parse_args(
["filters", self.testInput0, "c", "d", "--", "f"]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "filters")
self.assertEqual(parsed.test[0].name, self.testInput0)
self.assertEqual(parsed.filter_baseline[0], "c")
self.assertEqual(parsed.filter_contender[0], "d")
self.assertEqual(parsed.benchmark_options, ["f"])
def test_benchmarksfiltered_basic(self):
parsed = self.parser.parse_args(
["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "benchmarksfiltered")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.filter_baseline[0], "c")
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertEqual(parsed.filter_contender[0], "e")
self.assertFalse(parsed.benchmark_options)
def test_benchmarksfiltered_with_remainder(self):
parsed = self.parser.parse_args(
[
"benchmarksfiltered",
self.testInput0,
"c",
self.testInput1,
"e",
"f",
]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "benchmarksfiltered")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.filter_baseline[0], "c")
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertEqual(parsed.filter_contender[0], "e")
self.assertEqual(parsed.benchmark_options[0], "f")
def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
parsed = self.parser.parse_args(
[
"benchmarksfiltered",
self.testInput0,
"c",
self.testInput1,
"e",
"--",
"g",
]
)
self.assertFalse(parsed.display_aggregates_only)
self.assertTrue(parsed.utest)
self.assertEqual(parsed.mode, "benchmarksfiltered")
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
self.assertEqual(parsed.filter_baseline[0], "c")
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
self.assertEqual(parsed.filter_contender[0], "e")
self.assertEqual(parsed.benchmark_options[0], "g")
if __name__ == "__main__":
# unittest.main()
main()
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
# kate: indent-mode python; remove-trailing-spaces modified;
Codebase Browser
llvm