# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from collections import Counter
import json
import logging
import os
import subprocess
import traceback
from .file_util import (GetBenchmarkDBSize, GetDataDir,
GetExpectedHistogramsDictionary,
GetExpectedHistogramsFile, GetHistogramCountDeltasFile,
GetProcessedFiles, GetProcessor, GetRawHistogramsFile,
GetRunPathFile, MovePreviousHistogramsFiles)
from .histogram_list import GetSharedStorageUmaHistograms
from .util import GetNonePlaceholder, JsonDump
def _ProcessHistograms(histogram_data):
if not isinstance(histogram_data, list):
raise RuntimeError(
'Expected histogram_data to be type list but got type %s' %
type(histogram_data))
guid_map = {}
histogram_info = {}
expected_processed_all_guids = False
benchmark_name = None
for item in histogram_data:
if not isinstance(item, dict):
raise RuntimeError('Expected item to be type dict but got type %s' %
type(item))
if 'guid' in item:
if expected_processed_all_guids:
info = ('; %s\nitem[`guid`]: %s' %
(JsonDump(item), item.get('guid', '[[Not Found]]')))
msg = 'Expected to process all items with `guid` before any with `name`'
raise RuntimeError(msg + info)
if len(item) != 3:
raise RuntimeError(
'Expected item with `guid` to have length 3; got item: %s' %
JsonDump(item))
if 'type' not in item:
raise RuntimeError(
'Expected item with `guid` to have key `type`; got: %s' %
JsonDump(item))
[value_key] = [key for key in item if key not in ('guid', 'type')]
guid_map[item['guid']] = item[value_key]
elif 'name' in item:
expected_processed_all_guids = True
name = item['name']
if not name in GetSharedStorageUmaHistograms():
continue
for key in ['diagnostics', 'running', 'sampleValues']:
if key not in item:
raise RuntimeError(
'Expected item with `name` to have key `%s`; got: %s' %
(key, JsonDump(item)))
histogram_info.setdefault(name, [])
item_diagnostics = item['diagnostics']
diagnostics = {
key: guid_map.get(item_diagnostics[key], item_diagnostics[key])
for key in item_diagnostics
}
item_info = {
'diagnostics': diagnostics,
'running': item['running'],
'sampleValues': item['sampleValues']
}
histogram_info[name].append(item_info)
if not benchmark_name:
if (not 'benchmarks' in diagnostics
or len(diagnostics['benchmarks']) != 1):
RuntimeError(
'Expected `diagnostics` to have `benchmarks` of length 1; got %s'
% JsonDump(diagnostics))
benchmark_name = diagnostics['benchmarks'][0]
agg_histograms = {}
by_story = {}
counts = {}
for histogram, info_list in histogram_info.items():
agg_histograms.setdefault(histogram, [])
for entry in info_list:
for key in ['diagnostics', 'sampleValues']:
if key not in entry:
raise RuntimeError('Entry %s does not have key `%s`' %
(JsonDump(entry), key))
for key in ['stories', 'storysetRepeats']:
if key not in entry['diagnostics']:
raise RuntimeError('Entry diagnostics %s does not have key `%s`' %
(JsonDump(entry), key))
if len(entry['diagnostics'][key]) != 1:
raise RuntimeError(
'Entry diagnostics %s expected `%s` to lave length 1' %
(JsonDump(entry), key))
agg_histograms[histogram] += entry['sampleValues']
[story] = entry['diagnostics']['stories']
[rep] = entry['diagnostics']['storysetRepeats']
by_story.setdefault(story, {}).setdefault(rep,
{}).setdefault(histogram, [])
by_story[story][rep][histogram] += entry['sampleValues']
counts.setdefault(story, {}).setdefault(rep, Counter())
counts[story][rep][histogram] += len(entry['sampleValues'])
runs_all_match, deltas = _CheckHistogramCounts(counts)
results = [histogram_info, agg_histograms, by_story, counts, deltas]
size = GetBenchmarkDBSize(benchmark_name)
return runs_all_match, size, list(map(JsonDump, results))
def _CheckHistogramCounts(actual_counts):
counts_match = True
expected_counts = GetExpectedHistogramsDictionary()
missed_stories = (set(expected_counts.keys()).difference(
set(actual_counts.keys())))
if len(missed_stories) > 0:
counts_match = False
logging.warning('Missed recording for stories %s' %
repr(list(missed_stories)))
unexpected_stories = (set(actual_counts.keys()).difference(
set(expected_counts.keys())))
if len(missed_stories) > 0:
counts_match = False
logging.warning('Unexpectedly recorded for stories %s' %
repr(list(unexpected_stories)))
deltas = {}
for story, repeats in actual_counts.items():
for repeat, histogram_counts in repeats.items():
if histogram_counts == expected_counts[story]:
continue
counts_match = False
deltas.setdefault(story, {}).setdefault(repeat, Counter())
missed_histograms = (set(expected_counts[story].keys()).difference(
set(histogram_counts.keys())))
for histogram in missed_histograms:
expected_count = expected_counts[story][histogram]
deltas[story][repeat][histogram] = -expected_count
for histogram, count in histogram_counts.items():
expected_count = expected_counts[story].get(histogram, 0)
if count != expected_count:
deltas[story][repeat][histogram] = count - expected_count
logging.warning(
'Story %s, repeat %s failed to record expected histogram counts' %
(story, repeat))
logging.warning('Expected counts: %s' % JsonDump(expected_counts[story]))
logging.warning('Actual counts: %s' % JsonDump(histogram_counts))
return counts_match, deltas
def _ProcessResultsInternal():
if not os.path.exists(GetDataDir()) or not os.path.isdir(GetDataDir()):
raise RuntimeError(
'Data directory %s does not exist or is not a directory' %
(GetDataDir()))
if (not os.path.exists(GetRunPathFile())
or not os.path.isfile(GetRunPathFile())):
raise RuntimeError('Run path file %s does not exist or is not a file' %
GetRunPathFile())
run_results_path = None
with open(GetRunPathFile(), 'r') as f:
run_results_path = f.read()
none_placeholder = GetNonePlaceholder()
if not run_results_path or run_results_path.find(none_placeholder) > -1:
raise RuntimeError('Directory path of perf results not found')
MovePreviousHistogramsFiles()
subprocess.check_call([
GetProcessor(), "--output-format=histograms",
"--intermediate-dir=" + run_results_path
])
if (not os.path.exists(GetRawHistogramsFile())
or not os.path.isfile(GetRawHistogramsFile())):
raise RuntimeError('Histogram results %s not found or is not a file' %
GetRawHistogramsFile())
histogram_data = None
with open(GetRawHistogramsFile(), 'r') as f:
histogram_data = json.load(f)
runs_all_match, size, results = _ProcessHistograms(histogram_data)
print("Benchmark's initial database size: %s" % size)
if runs_all_match:
print('SUCCESS: Actual histogram counts are equal to expected counts ' +
'for all stories and repeats.')
else:
print('WARNING: Actual histogram counts differ from expected counts for ' +
'at least one story/repeat. See file://%s for the deltas.' %
GetHistogramCountDeltasFile())
print('View expected histogram counts at file://%s' %
GetExpectedHistogramsFile())
for (result, filename) in zip(results, GetProcessedFiles()):
if filename == GetHistogramCountDeltasFile():
json_result = json.loads(result)
if runs_all_match and len(json_result) == 0:
continue
if runs_all_match:
logging.warning(('Unexpectedly got histogram count deltas %s when ' %
result) +
'all runs were reported to have counts match')
elif len(json_result) == 0:
logging.warning(
'A discrepancy in run counts was reported, but the returned ' +
'deltas object was empty')
with open(filename, 'w') as f:
f.write(result)
print('View processed results at file://%s' % filename)
def ProcessResults():
try:
_ProcessResultsInternal()
return 0
except Exception as e: # pylint: disable=broad-except
logging.warning('Encountered exception: %s\n%s' %
(repr(e), traceback.format_exc()))
return 1
Codebase Browser
chromium