# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Please see the comments section at the top of the `run_performance_tests.py`.
import json
import os
import pathlib
import shutil
import tempfile
import unittest
from unittest import mock
import run_performance_tests
from run_performance_tests import TelemetryCommandGenerator
from telemetry.internal.util import binary_manager
# Protected access is allowed for unittests.
# pylint: disable=protected-access
# The path where the output of a wpt run was written. This is the file that
# gets processed by BaseWptScriptAdapter.
OUTPUT_JSON_FILENAME = 'out.json'
class TelemetryCommandGeneratorTest(unittest.TestCase):
def setUp(self):
fake_args = ['./run_benchmark', '--isolated-script-test-output=output.json']
self._fake_options = run_performance_tests.parse_arguments(fake_args)
mock.patch.object(binary_manager, 'InitDependencyManager').start()
def testStorySelectionBeginEnd(self):
story_selection_config = json.loads(
'{"begin": 11, "end": 21, "abridged": false}')
generator = TelemetryCommandGenerator('benchmark_name', self._fake_options,
story_selection_config)
command = generator.generate('output_dir')
self.assertIn('--story-shard-begin-index=11', command)
self.assertIn('--story-shard-end-index=21', command)
self.assertNotIn('--run-abridged-story-set', command)
def testStorySelectionAbridgedDefault(self):
story_selection_config = json.loads('{"begin": 11, "end": 21}')
generator = TelemetryCommandGenerator('benchmark_name', self._fake_options,
story_selection_config)
command = generator.generate('output_dir')
self.assertIn('--run-abridged-story-set', command)
def testStorySelectionIndexSectionsSingleIndex(self):
story_selection_config = json.loads(
'{"sections": [{"begin": 11, "end": 21}, {"begin": 25, "end": 26}]}')
generator = TelemetryCommandGenerator('benchmark_name', self._fake_options,
story_selection_config)
command = generator.generate('output_dir')
self.assertIn('--story-shard-indexes=11-21,25', command)
def testStorySelectionIndexSectionsOpenEnds(self):
story_selection_config = json.loads(
'{"sections": [{"end": 10}, {"begin": 15, "end": 16}, {"begin": 20}]}')
generator = TelemetryCommandGenerator('benchmark_name', self._fake_options,
story_selection_config)
command = generator.generate('output_dir')
self.assertIn('--story-shard-indexes=-10,15,20-', command)
def testStorySelectionIndexSectionsIllegalRange(self):
with self.assertRaises(ValueError):
story_selection_config = json.loads(
'{"sections": [{"begin": 15, "end": 16}, {"foo": "bar"}]}')
generator = TelemetryCommandGenerator('benchmark_name',
self._fake_options,
story_selection_config)
generator.generate('output_dir')
def testStorySelectionIndexSectionsEmpty(self):
story_selection_config = json.loads('{"sections": []}')
generator = TelemetryCommandGenerator('benchmark_name', self._fake_options,
story_selection_config)
command = generator.generate('output_dir')
self.assertNotIn('--story-shard-indexes=', command)
@mock.patch.object(os.path, 'exists')
@mock.patch.object(run_performance_tests, 'copy_map_file_to_out_dir')
@mock.patch('builtins.open',
new_callable=mock.mock_open,
read_data='{"foo": 1}')
def testLoadMapFileSuccess(self, mock_open, mock_copy_map_file_to_out_dir,
mock_exists):
del mock_open, mock_exists
content = run_performance_tests.load_map_file('file', 'dir')
self.assertTrue(isinstance(content, dict))
self.assertEqual(content['foo'], 1)
mock_copy_map_file_to_out_dir.assert_called_with('file', 'dir')
@mock.patch.object(os.path, 'exists')
@mock.patch.object(pathlib.Path, 'exists')
@mock.patch.object(run_performance_tests, 'copy_map_file_to_out_dir')
@mock.patch('builtins.open',
new_callable=mock.mock_open,
read_data='{"foo": 1}')
def testLoadMapFileShardMapDirectory(self, mock_open,
mock_copy_map_file_to_out_dir,
mock_pathlib_exists, mock_exists):
del mock_open
mock_exists.return_value = False
mock_pathlib_exists.return_value = True
expected_file = str(run_performance_tests.SHARD_MAPS_DIR / 'file')
run_performance_tests.load_map_file('file', 'dir')
mock_copy_map_file_to_out_dir.assert_called_with(expected_file, 'dir')
@mock.patch.object(os.path, 'exists')
@mock.patch.object(run_performance_tests, 'copy_map_file_to_out_dir')
@mock.patch('builtins.open',
new_callable=mock.mock_open,
read_data='{"foo": 1}')
def testLoadMapFileException(self, mock_open, mock_copy_map_file_to_out_dir,
mock_exists):
del mock_open, mock_copy_map_file_to_out_dir
mock_exists.side_effect = [False, False]
with self.assertRaises(Exception):
run_performance_tests.load_map_file('file', 'dir')
@mock.patch.object(run_performance_tests, 'copy_map_file_to_out_dir')
@mock.patch.object(tempfile, 'NamedTemporaryFile')
def testLoadMapStringSuccess(self, mock_named_temporary_file,
mock_copy_map_file_to_out_dir):
del mock_named_temporary_file
content = run_performance_tests.load_map_string('{"foo": 1}', 'dir')
self.assertTrue(isinstance(content, dict))
self.assertEqual(content['foo'], 1)
mock_copy_map_file_to_out_dir.assert_called_with(mock.ANY, 'dir')
@mock.patch.object(os.path, 'exists')
@mock.patch.object(shutil, 'copyfile')
def testCopyMapFileToOutDirSuccess(self, mock_copyfile, mock_exists):
del mock_exists
run_performance_tests.copy_map_file_to_out_dir('file', 'dir')
mock_copyfile.assert_called_with('file', 'dir/benchmarks_shard_map.json')
@mock.patch.object(run_performance_tests.CrossbenchTest, 'execute_benchmark')
def testCrossbenchTestBenchmarksArg(self, mock_execute_benchmark):
fake_args = self._create_crossbench_args()
options = run_performance_tests.parse_arguments(fake_args)
run_performance_tests.CrossbenchTest(options, 'dir').execute()
mock_execute_benchmark.assert_called_with('speedometer_3.0',
'speedometer3.crossbench', [])
def testCrossbenchTestBenchmarksException(self):
fake_args = ['./cp.py', '--isolated-script-test-output=output']
options = run_performance_tests.parse_arguments(fake_args)
with self.assertRaises(Exception):
run_performance_tests.CrossbenchTest(options, 'dir').execute()
def testCrossbenchTestMultiBenchmarksException(self):
fake_args = [
'./cp.py', '--isolated-script-test-output=output',
'--benchmarks=speedometer_3.0,speedometer_2.0'
]
options = run_performance_tests.parse_arguments(fake_args)
with self.assertRaises(Exception):
run_performance_tests.CrossbenchTest(options, 'dir').execute()
@mock.patch.object(run_performance_tests, '_run_benchmarks_on_shardmap')
@mock.patch.object(os.path, 'dirname')
@mock.patch.object(run_performance_tests, 'load_map_file')
def testCrossbenchTestShardMapFile(self, mock_load_map_file, mock_direname,
mock_run_benchmarks_on_shardmap):
mock_load_map_file.return_value = 'map_file'
mock_direname.return_value = 'dir'
fake_args = [
'skip', 'run_benchmark', '--isolated-script-test-output=output',
'--test-shard-map-filename=foo'
]
expected_options = run_performance_tests.parse_arguments(fake_args[1:])
run_performance_tests.main(fake_args)
mock_load_map_file.assert_called_with('foo', 'dir')
mock_run_benchmarks_on_shardmap.assert_called_with('map_file',
expected_options, 'dir',
[])
@mock.patch.object(run_performance_tests, '_run_benchmarks_on_shardmap')
@mock.patch.object(os.path, 'dirname')
@mock.patch.object(run_performance_tests, 'load_map_string')
def testCrossbenchTestShardMapString(self, mock_load_map_string,
mock_direname,
mock_run_benchmarks_on_shardmap):
mock_load_map_string.return_value = 'map_string'
mock_direname.return_value = 'dir'
fake_args = [
'skip', 'run_benchmark', '--isolated-script-test-output=output',
'--use-dynamic-shards', '--dynamic-shardmap=json'
]
expected_options = run_performance_tests.parse_arguments(fake_args[1:])
run_performance_tests.main(fake_args)
mock_load_map_string.assert_called_with('json', 'dir')
mock_run_benchmarks_on_shardmap.assert_called_with('map_string',
expected_options, 'dir',
[])
@mock.patch.object(run_performance_tests.CrossbenchTest, 'execute_benchmark')
@mock.patch.dict(os.environ, {'GTEST_SHARD_INDEX': '0'})
def testCrossbenchTestRunBenchmarkOnShardMap(self, mock_execute_benchmark):
fake_args = [
'run_benchmark',
'--isolated-script-test-output=output',
'--test-shard-map-filename=foo',
'--browser=./chrome',
]
options = run_performance_tests.parse_arguments(fake_args)
shard_map = {
'0': {
'crossbench': {
'my_benchmark': {
'display_name': 'my_display',
'arguments': []
}
}
}
}
mock_execute_benchmark.return_value = 0
return_code = run_performance_tests._run_benchmarks_on_shardmap(
shard_map, options, 'dir', [])
self.assertEqual(return_code, 0)
mock_execute_benchmark.assert_called_with('my_benchmark', 'my_display', [])
@mock.patch.object(run_performance_tests.CrossbenchTest, 'execute_benchmark')
def testCrossbenchTestMissingShardIndex(self, mock_execute_benchmark):
del mock_execute_benchmark
fake_args = [
'run_benchmark', '--isolated-script-test-output=output',
'--test-shard-map-filename=foo'
]
options = run_performance_tests.parse_arguments(fake_args)
shard_map = {'0': {'crossbench': {'my_benchmark': []}}}
with self.assertRaises(Exception):
run_performance_tests._run_benchmarks_on_shardmap(shard_map, options,
'dir', [])
@mock.patch.object(run_performance_tests.CrossbenchTest, 'execute_benchmark')
@mock.patch.dict(os.environ, {'GTEST_SHARD_INDEX': '0'})
def testCrossbenchTestMissingBenchmark(self, mock_execute_benchmark):
fake_args = [
'run_benchmark',
'--isolated-script-test-output=output',
'--test-shard-map-filename=foo',
'--browser=./chrome',
]
options = run_performance_tests.parse_arguments(fake_args)
shard_map = {'0': {'crossbench': {}}}
return_code = run_performance_tests._run_benchmarks_on_shardmap(
shard_map, options, 'dir', [])
self.assertEqual(return_code, 0)
mock_execute_benchmark.assert_not_called()
@mock.patch.object(run_performance_tests.CrossbenchTest, 'execute_benchmark')
@mock.patch.dict(os.environ, {'GTEST_SHARD_INDEX': '0'})
def testCrossbenchTestRunMultiBenchmarkOnShardMap(self,
mock_execute_benchmark):
fake_args = [
'run_benchmark',
'--isolated-script-test-output=output',
'--test-shard-map-filename=foo',
'--browser=./chrome',
]
options = run_performance_tests.parse_arguments(fake_args)
shard_map = {
'0': {
'crossbench': {
'b1': {
'display_name': 'display1',
'arguments': []
},
'b2': {
'display_name': 'display2',
'arguments': []
}
}
}
}
mock_execute_benchmark.return_value = 1
return_code = run_performance_tests._run_benchmarks_on_shardmap(
shard_map, options, 'dir', [])
self.assertEqual(return_code, 1)
mock_execute_benchmark.assert_has_calls(
[mock.call('b1', 'display1', []),
mock.call('b2', 'display2', [])])
def testCrossbenchGetNetworkArgWithNetwork(self):
fake_args = self._create_crossbench_args() + ['--network=foo']
options = run_performance_tests.parse_arguments(fake_args)
expected_network = ['--network=foo']
crosebench_test = run_performance_tests.CrossbenchTest(options, 'dir')
self.assertEqual(crosebench_test.network, expected_network)
def testCrossbenchGetDefaultFileServer(self):
fake_args = self._create_crossbench_args() + ['--fileserver']
options = run_performance_tests.parse_arguments(fake_args)
src_dir = run_performance_tests.CHROMIUM_SRC_DIR
local_fileserver = str(src_dir / 'third_party/speedometer/v3.0')
expected_dict = {
'type': 'local',
'path': local_fileserver,
'url': 'http://localhost:8000'
}
crosebench_test = run_performance_tests.CrossbenchTest(options, 'dir')
network_dict = json.loads(crosebench_test.network[0].split('=', 1)[1])
self.assertDictEqual(network_dict, expected_dict)
def testCrossbenchGetTargetFileServer(self):
fake_args = self._create_crossbench_args() + ['--fileserver=foo']
options = run_performance_tests.parse_arguments(fake_args)
src_dir = run_performance_tests.CHROMIUM_SRC_DIR
local_fileserver = str(src_dir / 'foo')
expected_dict = {
'type': 'local',
'path': local_fileserver,
'url': 'http://localhost:8000'
}
crosebench_test = run_performance_tests.CrossbenchTest(options, 'dir')
network_dict = json.loads(crosebench_test.network[0].split('=', 1)[1])
self.assertDictEqual(network_dict, expected_dict)
@mock.patch.object(binary_manager, 'FetchPath')
def testCrossbenchGetDefaultWpr(self, mock_fetch_path):
mock_fetch_path.return_value = 'wpr_go_path'
fake_args = self._create_crossbench_args() + ['--wpr']
options = run_performance_tests.parse_arguments(fake_args)
data_dir = run_performance_tests.PAGE_SETS_DATA
archive = str(data_dir / 'crossbench_android_speedometer_3.0_000.wprgo')
expected_dict = {
'type': 'wpr',
'path': archive,
'wpr_go_bin': 'wpr_go_path'
}
crosebench_test = run_performance_tests.CrossbenchTest(options, 'dir')
network_dict = json.loads(crosebench_test.network[0].split('=', 1)[1])
self.assertDictEqual(network_dict, expected_dict)
@mock.patch.object(binary_manager, 'FetchPath')
def testCrossbenchGetTargetWpr(self, mock_fetch_path):
mock_fetch_path.return_value = 'wpr_go_path'
fake_args = self._create_crossbench_args() + ['--wpr=foo']
options = run_performance_tests.parse_arguments(fake_args)
data_dir = run_performance_tests.PAGE_SETS_DATA
archive = str(data_dir / 'foo')
expected_dict = {
'type': 'wpr',
'path': archive,
'wpr_go_bin': 'wpr_go_path'
}
crosebench_test = run_performance_tests.CrossbenchTest(options, 'dir')
network_dict = json.loads(crosebench_test.network[0].split('=', 1)[1])
self.assertDictEqual(network_dict, expected_dict)
def _create_crossbench_args(self, browser='./chrome'):
return [
'./cp.py',
'--isolated-script-test-output=output',
'--benchmarks=speedometer_3.0',
'--benchmark-display-name=speedometer3.crossbench',
f'--browser={browser}',
]
Codebase Browser
chromium