# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import shared_page_state
from telemetry import benchmark
from telemetry import story
from page_sets.rendering import rendering_story
from page_sets.rendering import story_tags
from page_sets.system_health import platforms
class MotionMarkPage(rendering_story.RenderingStory):
ABSTRACT_STORY = True
TAGS = [story_tags.MOTIONMARK]
SUPPORTED_PLATFORMS = platforms.MOBILE_ONLY
EXTRA_BROWSER_ARGS = None
def __init__(self,
page_set,
shared_page_state_class=shared_page_state.SharedMobilePageState,
name_suffix='',
extra_browser_args=None):
extra_browser_args = ['--report-silk-details', '--disable-top-sites']
if self.EXTRA_BROWSER_ARGS is not None:
extra_browser_args.append(self.EXTRA_BROWSER_ARGS)
super(MotionMarkPage,
self).__init__(page_set=page_set,
shared_page_state_class=shared_page_state_class,
name_suffix=name_suffix,
make_javascript_deterministic=False,
extra_browser_args=extra_browser_args)
self._score = 0
self._scoreLowerBound = 0
self._scoreUpperBound = 0
self._stories = []
self._storyScores = []
self._storyScoreLowerBounds = []
self._storyScoreUpperBounds = []
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self.url)
action_runner.WaitForJavaScriptCondition(
'document.readyState == "complete"')
def RunPageInteractions(self, action_runner):
action_runner.Wait(3)
with action_runner.CreateInteraction('Filter'):
action_runner.Wait(20)
action_runner.WaitForJavaScriptCondition(
'window.benchmarkRunnerClient.results._results')
[score, lower, upper] = action_runner.EvaluateJavaScript(
'''[window.benchmarkRunnerClient.results.score,
window.benchmarkRunnerClient.results.scoreLowerBound,
window.benchmarkRunnerClient.results.scoreUpperBound]''')
self._score = score
self._scoreLowerBound = lower
self._scoreUpperBound = upper
# Navigate to about:blank to stop rendering frames and let the device
# cool down while the trace data for the story is processed.
action_runner.Navigate('about:blank')
@property
def score(self):
return self._score
@property
def scoreLowerBound(self):
return self._scoreLowerBound
@property
def scoreUpperBound(self):
return self._scoreUpperBound
@property
def stories(self):
return self._stories
@property
def storyScores(self):
return self._storyScores
@property
def storyScoreLowerBounds(self):
return self._storyScoreLowerBounds
@property
def storyScoreUpperBounds(self):
return self._storyScoreUpperBounds
@classmethod
def GetUrl(cls, suite_name, test_name, complexity):
# Strip unwanted characters from names
for ch in [' ', '.', ',']:
suite_name = suite_name.replace(ch, '')
test_name = test_name.replace(ch, '')
# Using latest MotionMark 1.3.1 as previous patches from 1.3 were pulled
# into the main repo. Commit at time of recording
# https://github.com/WebKit/MotionMark/commit/be2a5fea89b6ef411b053ebeb95a6302b3dc0ecb
return ('https://browserbench.org/MotionMark1.3.1/developer.html'
'?suite-name=%s'
'&test-name=%s'
'&complexity=%d'
'&test-interval=20'
'&warmup-length=2000'
'&warmup-frame-count=30'
'&first-frame-minimum-length=0'
'&display=minimal'
'&tiles=big'
'&controller=fixed'
'&system-frame-rate=60'
'&frame-rate=60'
'&time-measurement=performance') % (suite_name, test_name,
complexity)
class MotionMarkRampPage(MotionMarkPage):
ABSTRACT_STORY = True
TAGS = [story_tags.MOTIONMARK, story_tags.MOTIONMARK_RAMP]
SUPPORTED_PLATFORMS = platforms.ALL_PLATFORMS
@classmethod
def GetRampUrl(cls, suite_name, test_name):
# Strip unwanted characters from names
for ch in [' ', '.', ',']:
suite_name = suite_name.replace(ch, '')
test_name = test_name.replace(ch, '')
# Using latest MotionMark 1.3.1 as previous patches from 1.3 were pulled
# into the main repo. Commit at time of recording
# https://github.com/WebKit/MotionMark/commit/be2a5fea89b6ef411b053ebeb95a6302b3dc0ecb
return ('https://browserbench.org/MotionMark1.3.1/developer.html'
'?suite-name=%s'
'&test-name=%s'
'&test-interval=20'
'&display=minimal'
'&tiles=big'
'&controller=ramp'
'&system-frame-rate=60'
'&frame-rate=60'
'&warmup-length=2000'
'&warmup-frame-count=30'
'&time-measurement=performance') % (suite_name, test_name)
class MotionMarkRampMultiply(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_multiply'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Multiply')
class MotionMarkRampCanvasArcs(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_canvas_arcs'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Canvas Arcs')
class MotionMarkRampLeaves(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_leaves'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Leaves')
class MotionMarkRampPaths(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_paths'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Paths')
class MotionMarkRampCanvasLines(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_canvas_lines'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Canvas Lines')
class MotionMarkRampImages(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_images'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Images')
class MotionMarkRampDesign(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_design'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Design')
class MotionMarkRampSuits(MotionMarkRampPage):
BASE_NAME = 'motionmark_ramp_suits'
URL = MotionMarkRampPage.GetRampUrl('MotionMark', 'Suits')
class MotionMarkRampComposite(MotionMarkPage):
DISABLE_TRACING = True
TAGS = [story_tags.MOTIONMARK, story_tags.MOTIONMARK_RAMP]
SUPPORTED_PLATFORMS = platforms.ALL_PLATFORMS
BASE_NAME = 'motionmark_ramp_composite'
# Using latest MotionMark 1.3.1 as previous patches from 1.3 were pulled
# into the main repo. Commit at time of recording
# https://github.com/WebKit/MotionMark/commit/be2a5fea89b6ef411b053ebeb95a6302b3dc0ecb
URL = 'https://browserbench.org/MotionMark1.3.1/developer.html'
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self.url)
action_runner.Wait(3)
action_runner.ExecuteJavaScript('''
const list = document.querySelectorAll('.tree > li');
const row = list[0];
const labels = row.querySelectorAll('input[type=checkbox]');
for (const label of labels) {
label.checked = true;
}
''')
action_runner.ExecuteJavaScript(
'window.benchmarkController.startBenchmark()')
action_runner.WaitForJavaScriptCondition(
'document.readyState == "complete"')
def RunPageInteractions(self, action_runner):
action_runner.Wait(3)
with action_runner.CreateInteraction('Filter'):
action_runner.Wait(300) # Determined experimentally
action_runner.WaitForJavaScriptCondition(
'window.benchmarkRunnerClient.results._results')
[score, lower, upper] = action_runner.EvaluateJavaScript(
'''[window.benchmarkRunnerClient.results.score,
window.benchmarkRunnerClient.results.scoreLowerBound,
window.benchmarkRunnerClient.results.scoreUpperBound]''')
self._score = score
self._scoreLowerBound = lower
self._scoreUpperBound = upper
# The MotionMark object is a non-iterable map, so we need to access the
# components manually. Currently we only run one iteration, this would
# need to be updated if we add iteration support in the future.
[stories, scores, lowerBounds,
upperBounds] = action_runner.EvaluateJavaScript('''const stories =
window.benchmarkRunnerClient.results._results.
iterationsResults[0].testsResults.MotionMark;
const scores = [];
const lowerBounds = [];
const upperBounds = [];
for (const val of Object.keys(stories)) {
const story = stories[val];
scores.push(story.score);
lowerBounds.push(story.scoreLowerBound);
upperBounds.push(story.scoreUpperBound);
}
[stories, scores, lowerBounds, upperBounds]''')
self._stories = stories
self._storyScores = scores
self._storyScoreLowerBounds = lowerBounds
self._storyScoreUpperBounds = upperBounds
# Navigate to about:blank to stop rendering frames and let the device
# cool down while the trace data for the story is processed.
action_runner.Navigate('about:blank')
def WillStartTracing(self, chrome_trace_config):
chrome_trace_config.record_mode = 'record-until-full'
class MotionMarkFixed2SecondsPage(MotionMarkPage):
ABSTRACT_STORY = True
TAGS = [story_tags.MOTIONMARK, story_tags.MOTIONMARK_FIXED_2_SECONDS]
SUPPORTED_PLATFORMS = platforms.ALL_PLATFORMS
def RunPageInteractions(self, action_runner):
with action_runner.CreateInteraction('Filter'):
action_runner.WaitForJavaScriptCondition(
'window.benchmarkRunnerClient.results._results')
@classmethod
def GetFixed2SecondsUrl(cls, suite_name, test_name, complexity,
test_interval):
# Strip unwanted characters from names
for ch in [' ', '.', ',']:
suite_name = suite_name.replace(ch, '')
test_name = test_name.replace(ch, '')
# Using latest MotionMark 1.3.1 as previous patches from 1.3 were pulled
# into the main repo. Commit at time of recording#
# https://github.com/WebKit/MotionMark/commit/be2a5fea89b6ef411b053ebeb95a6302b3dc0ecb
return ('https://browserbench.org/MotionMark1.3.1/developer.html'
'?suite-name=%s'
'&test-name=%s'
'&complexity=%d'
'&test-interval=%d'
'&warmup-length=0'
'&warmup-frame-count=0'
'&first-frame-minimum-length=0'
'&display=minimal'
'&tiles=big'
'&controller=fixed'
'&system-frame-rate=60'
'&frame-rate=60'
'&time-measurement=performance') % (suite_name, test_name,
complexity, test_interval)
# Numbers for complexity come from recent high scores on perf bots with a PGO
# build.
#TODO(vmiura): Update names from fixed_2_seconds to match the new durations.
class MotionMarkFixed2SecondsMultiply(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_multiply'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark',
'Multiply', 5150, 5)
class MotionMarkFixed2SecondsCanvasArcs(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_canvas_arcs'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark',
'Canvas Arcs', 17400, 5)
class MotionMarkFixed2SecondsLeaves(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_leaves'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark', 'Leaves',
4300, 5)
class MotionMarkFixed2SecondsPaths(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_paths'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark', 'Paths',
64700, 5)
class MotionMarkFixed2SecondsCanvasLines(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_canvas_lines'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark',
'Canvas Lines', 54200,
5)
class MotionMarkFixed2SecondsImages(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_images'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark', 'Images',
440, 5)
class MotionMarkFixed2SecondsDesign(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_design'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark', 'Design',
705, 5)
class MotionMarkFixed2SecondsSuits(MotionMarkFixed2SecondsPage):
BASE_NAME = 'motionmark_fixed_2_seconds_suits'
URL = MotionMarkFixed2SecondsPage.GetFixed2SecondsUrl('MotionMark', 'Suits',
2800, 5)
@benchmark.Info(emails=['[email protected]'],
component='Internals>GPU>Internals')
class MotionMarkRampCompositeGraphite(MotionMarkRampComposite):
BASE_NAME = 'motionmark_ramp_composite_graphite'
SUPPORTED_PLATFORMS = [story.expectations.ALL_MAC, story.expectations.ALL_WIN]
EXTRA_BROWSER_ARGS = '--enable-features=SkiaGraphite'
@benchmark.Info(emails=['[email protected]'],
component='Internals>GPU>Internals')
class MotionMarkRampCompositeGanesh(MotionMarkRampComposite):
BASE_NAME = 'motionmark_ramp_composite_ganesh'
SUPPORTED_PLATFORMS = [story.expectations.ALL_MAC, story.expectations.ALL_WIN]
EXTRA_BROWSER_ARGS = '--disable-features=SkiaGraphite'
Codebase Browser
chromium