# Copyright 2015 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from __future__ import print_function
import logging
import os
import sys
import time
from telemetry.core import exceptions
from telemetry.internal.results import artifact_compatibility_wrapper as acw
from telemetry.internal.results import artifact_logger
from telemetry.testing import tab_test_case
from telemetry import decorators
import py_utils
# Possible ways that gl::Crash() will show up in a stack trace.
GPU_CRASH_SIGNATURES = [
'gl::Crash',
'chrome!Crash',
'GpuServiceImpl::Crash()',
]
# Possible ways that a renderer process crash intentionally caused by DevTools
# can show up in a stack trace.
FORCED_RENDERER_CRASH_SIGNATURES = [
'base::debug::BreakDebugger',
'blink::DevToolsSession::IOSession::DispatchProtocolCommand',
'blink::HandleChromeDebugURL',
'chrome!DispatchProtocolCommand',
'logging::LogMessage::~LogMessage',
]
def ContainsAtLeastOne(expected_values, checked_value):
for expected in expected_values:
if expected in checked_value:
return True
return False
class BrowserMinidumpTest(tab_test_case.TabTestCase):
def setUp(self):
# If something is wrong with minidump symbolization, we want to get all the
# debugging information we can from the bots since it may be difficult to
# reproduce the issue locally. So, use the full logger implementation.
artifact_logger.RegisterArtifactImplementation(
acw.FullLoggingArtifactImpl())
super(BrowserMinidumpTest, self).setUp()
def tearDown(self):
super(BrowserMinidumpTest, self).tearDown()
artifact_logger.RegisterArtifactImplementation(None)
def assertContainsAtLeastOne(self, expected_values, checked_value):
self.assertTrue(ContainsAtLeastOne(expected_values, checked_value),
'None of %s found in %s' % (expected_values, checked_value))
@decorators.Isolated
# Minidump symbolization doesn't work in ChromeOS local mode if the rootfs is
# still read-only, so skip the test in that case.
@decorators.Disabled(
'chromeos-local',
'win7', # https://crbug.com/1084931
)
def testSymbolizeMinidump(self):
# Wait for the browser to restart fully before crashing
self._LoadPageThenWait('var sam = "car";', 'sam')
self._browser.tabs.New().Navigate('chrome://gpucrash', timeout=10)
crash_minidump_path = self._browser.GetRecentMinidumpPathWithTimeout()
self.assertIsNotNone(crash_minidump_path)
if crash_minidump_path is not None:
logging.info('testSymbolizeMinidump: most recent path = '
+ crash_minidump_path)
all_paths = self._browser.GetAllMinidumpPaths()
if all_paths is not None:
logging.info('testSymbolizeMinidump: all paths ' + ''.join(all_paths))
all_unsymbolized_paths = self._browser.GetAllUnsymbolizedMinidumpPaths()
if all_unsymbolized_paths is not None:
logging.info('testSymbolizeMinidump: all unsymbolized paths '
+ ''.join(all_unsymbolized_paths))
# Flakes on chromeos: crbug.com/1014754
# This has failed to repro either locally or on swarming, so dump extra
# information if this is hit on the bots.
if len(all_unsymbolized_paths) != 1:
self._browser.CollectDebugData(logging.ERROR)
self.assertTrue(len(all_unsymbolized_paths) == 1)
# Now symbolize that minidump and make sure there are no longer any present
succeeded, stack = self._browser.SymbolizeMinidump(crash_minidump_path)
self.assertTrue(succeeded)
self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)
all_unsymbolized_after_symbolize_paths = \
self._browser.GetAllUnsymbolizedMinidumpPaths()
if all_unsymbolized_after_symbolize_paths is not None:
logging.info('testSymbolizeMinidump: after symbolize all '
+ 'unsymbolized paths: '
+ ''.join(all_unsymbolized_after_symbolize_paths))
self.assertTrue(len(all_unsymbolized_after_symbolize_paths) == 0)
@decorators.Isolated
# Minidump symbolization doesn't work in ChromeOS local mode if the rootfs is
# still read-only, so skip the test in that case.
@decorators.Disabled(
'chromeos-local',
'win7', # https://crbug.com/1084931
)
def testMultipleCrashMinidumps(self):
# Wait for the browser to restart fully before crashing
self._LoadPageThenWait('var cat = "dog";', 'cat')
self._browser.tabs.New().Navigate('chrome://gpucrash', timeout=10)
first_crash_path = self._browser.GetRecentMinidumpPathWithTimeout()
self.assertIsNotNone(first_crash_path)
if first_crash_path is not None:
logging.info('testMultipleCrashMinidumps: first crash most recent path ' +
first_crash_path)
all_paths = self._browser.GetAllMinidumpPaths()
if all_paths is not None:
logging.info('testMultipleCrashMinidumps: first crash all paths: '
+ ''.join(all_paths))
# Flakes on chromeos: crbug.com/1014754
# This has failed to repro either locally or on swarming, so dump extra
# information if this is hit on the bots.
if len(all_paths) != 1:
self._browser.CollectDebugData(logging.ERROR)
self.assertEqual(len(all_paths), 1)
self.assertEqual(all_paths[0], first_crash_path)
all_unsymbolized_paths = self._browser.GetAllUnsymbolizedMinidumpPaths()
self.assertTrue(len(all_unsymbolized_paths) == 1)
if all_unsymbolized_paths is not None:
logging.info('testMultipleCrashMinidumps: first crash all unsymbolized '
'paths: ' + ''.join(all_unsymbolized_paths))
# Restart the browser and then crash a second time
logging.info('Restarting the browser')
self._RestartBrowser()
# Start a new tab in the restarted browser
self._LoadPageThenWait('var foo = "bar";', 'foo')
self._browser.tabs.New().Navigate('chrome://gpucrash', timeout=10)
# Make the oldest allowable timestamp slightly after the first dump's
# timestamp so we don't get the first one returned to us again
oldest_ts = os.path.getmtime(first_crash_path) + 1
second_crash_path = self._browser.GetRecentMinidumpPathWithTimeout(
oldest_ts=oldest_ts)
self.assertIsNotNone(second_crash_path)
if second_crash_path is not None:
logging.info(
'testMultipleCrashMinidumps: second crash most recent path ' +
second_crash_path)
second_crash_all_paths = self._browser.GetAllMinidumpPaths()
if second_crash_all_paths is not None:
logging.info('testMultipleCrashMinidumps: second crash all paths: '
+ ''.join(second_crash_all_paths))
second_crash_all_unsymbolized_paths = \
self._browser.GetAllUnsymbolizedMinidumpPaths()
#self.assertTrue(len(all_unsymbolized_paths) == 1)
if second_crash_all_unsymbolized_paths is not None:
logging.info('testMultipleCrashMinidumps: second crash all unsymbolized '
'paths: ' + ''.join(second_crash_all_unsymbolized_paths))
self.assertEqual(len(second_crash_all_paths), 2)
# Check that both paths are now present and unsymbolized
self.assertTrue(first_crash_path in second_crash_all_paths)
self.assertTrue(second_crash_path in second_crash_all_paths)
self.assertTrue(len(second_crash_all_unsymbolized_paths) == 2)
# Now symbolize one of those paths and assert that there is still one
# unsymbolized
succeeded, stack = self._browser.SymbolizeMinidump(second_crash_path)
self.assertTrue(succeeded)
self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)
after_symbolize_all_paths = self._browser.GetAllMinidumpPaths()
if after_symbolize_all_paths is not None:
logging.info('testMultipleCrashMinidumps: after symbolize all paths: '
+ ''.join(after_symbolize_all_paths))
self.assertEqual(len(after_symbolize_all_paths), 2)
after_symbolize_all_unsymbolized_paths = \
self._browser.GetAllUnsymbolizedMinidumpPaths()
if after_symbolize_all_unsymbolized_paths is not None:
logging.info('testMultipleCrashMinidumps: after symbolize all '
+ 'unsymbolized paths: '
+ ''.join(after_symbolize_all_unsymbolized_paths))
self.assertEqual(after_symbolize_all_unsymbolized_paths, [first_crash_path])
# Explicitly ignore the remaining minidump so that it isn't detected during
# teardown by the test runner.
self._browser.IgnoreMinidump(first_crash_path)
@decorators.Isolated
# Minidump symbolization doesn't work in ChromeOS local mode if the rootfs is
# still read-only, so skip the test in that case.
@decorators.Disabled(
'chromeos-board-eve', # b/312565719
'chromeos-local',
'win7', # https://crbug.com/1084931
)
def testMinidumpFromRendererHang(self):
"""Tests that renderer hangs result in minidumps.
Telemetry has logic for detecting renderer hangs and killing the renderer
and GPU processes in such cases so we can get minidumps for diagnosing the
root cause.
"""
self._LoadPageThenWait('var cat = "dog";', 'cat')
try:
self._browser.tabs[-1].Navigate('chrome://hang', timeout=10)
except exceptions.Error:
# We expect the navigate to time out due to the hang.
pass
found_minidumps = False
try:
# Hung renderers are detected by JavaScript evaluation timing out, so
# try to evaluate something to trigger that.
# The timeout provided is the same one used for crashing the processes, so
# don't make it too short.
self._browser.tabs[-1].EvaluateJavaScript('var cat = "dog";', timeout=10)
except exceptions.TimeoutException:
# If we time out while crashing the renderer process, the minidump should
# still exist, we just have to manually look for it instead of it being
# part of the exception.
all_paths = self._browser.GetAllMinidumpPaths()
# We can't assert that we have exactly two minidumps because we can also
# get one from the renderer process being notified of the GPU process
# crash.
num_paths = len(all_paths)
self.assertTrue(num_paths in (2, 3),
'Got %d minidumps, expected 2 or 3' % num_paths)
found_renderer = False
found_gpu = False
for p in all_paths:
succeeded, stack = self._browser.SymbolizeMinidump(p)
self.assertTrue(succeeded)
try:
self.assertContainsAtLeastOne(FORCED_RENDERER_CRASH_SIGNATURES, stack)
# We don't assert that we haven't found a renderer crash yet since
# we can potentially get multiple under normal circumstances.
found_renderer = True
except AssertionError:
self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)
self.assertFalse(found_gpu, 'Found two GPU crashes')
found_gpu = True
self.assertTrue(found_renderer and found_gpu)
found_minidumps = True
except exceptions.AppCrashException as e:
self.assertTrue(e.is_valid_dump)
# We should get one minidump from the GPU process (gl::Crash()) and one
# minidump from the renderer process (base::debug::BreakDebugger()).
self.assertContainsAtLeastOne(FORCED_RENDERER_CRASH_SIGNATURES,
'\n'.join(e.stack_trace))
# There appears to be a bug on older versions of Windows 10 where the GPU
# minidump won't be found by the AppCrashException no matter how long we
# wait after the crash takes place. So, look for it afterwards.
if not ContainsAtLeastOne(GPU_CRASH_SIGNATURES, '\n'.join(e.stack_trace)):
self.assertEqual(sys.platform, 'win32')
minidumps = self._browser.GetAllUnsymbolizedMinidumpPaths()
self.assertEqual(len(minidumps), 1)
succeeded, stack = self._browser.SymbolizeMinidump(minidumps[0])
self.assertTrue(succeeded)
self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)
found_minidumps = True
self.assertTrue(found_minidumps)
def _LoadPageThenWait(self, script, value):
# We are occasionally seeing these tests fail on the first load and
# call to GetMostRecentMinidumpPath, where the directory is coming up empty.
# We are hypothesizing, that although the browser is technically loaded,
# some of chromes optimizations could still be running in the background
# that potentially initializing the crash directory that we set with the
# environment vairable BREAKPAD_DUMP_LOCATION in desktop_browser_backend.
# Therefore, we are adding a 5 second wait for now to see if this can help
# the problem until we determine if there is another chrome process we can
# wait on to ensure that all browser load processes have finished
time.sleep(5)
new_tab = self._browser.tabs.New()
new_tab.Navigate(self.UrlOfUnittestFile('blank.html'),
script_to_evaluate_on_commit=script)
# Wait until the javascript has run ensuring that
# the new browser has restarted before we crash it again
py_utils.WaitFor(lambda: new_tab.EvaluateJavaScript(value), 60)
Codebase Browser
chromium