#!/usr/bin/env python3
# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# This is a script helping developer bisect test failures.
#
# Currently this only supports bisecting gtest based test failures.
# Say you're assigned a BrowserTest.TestCase1 failure. You would generally do
# 1 Find a good commit and bad commit.
# 2 `git bisect start`
# 3 `git bisect good <good commit id>`
# 4 `git bisect bad <bad commit id>`
# 5 `gclient sync`
# 6 `autoninja -C out/Default browser_tests`
# 7 `out/Default/browser_tests --gtest_filter=BrowserTest.TestCase1`
# 8 if the test pass, `git bisect good`, otherwise `git bisect bad`.
# 9 repeat 5 - 8 until finding the culprit.
# This script will help you on 2 - 9. You first do 1, then run
# `python3 tools/bisect/bisect.py -g <good commit id> -b <bad commit id>
# --build_command 'autoninja -C out/Default browser_tests'
# --test_command 'out/Default/browser_tests
# --gtest_filter=BrowserTest.TestCase1'`
# The script will run until it finds the culprit cl breaking the test.
#
# Note1: We only support non-flaky -> failure, or non-flaky -> flaky.
# Flaky -> failure can't get correct result. For non-flaky -> flaky,
# you can use `--gtest_repeat`.
# Note2: For tests using python launching script, this is supported. e.g.
# `--test_command 'build/lacros/test_runner.py test
# out/lacrosdesktop/lacros_chrome_browsertests
# --ash-chrome-path=out/lacrosdesktop/ash_clang_x64/test_ash_chrome
# --gtest_filter=BrowserTest.TestCase1'`
import argparse
import subprocess
import sys
# This is the message from `git bisect` when it
# finds the culprit cl.
GIT_BAD_COMMENT_MSG = 'is the first bad commit'
GIT_BISECT_IN_PROCESS_MSG = 'left to test after this'
def Run(command, print_stdout_on_error=True):
print(command)
c = subprocess.run(command, shell=True)
if print_stdout_on_error and c.returncode != 0:
print(c.stdout)
return c.returncode == 0
def StartBisect(good_rev, bad_rev, build_command, test_command):
assert (Run('git bisect start'))
assert (Run('git bisect bad %s' % bad_rev))
assert (Run('git bisect good %s' % good_rev))
while True:
assert (Run('gclient sync'))
assert (Run(build_command))
test_ret = None
# If the test result is different running twice, then
# try again.
for _ in range(5):
c1 = Run(test_command, print_stdout_on_error=False)
c2 = Run(test_command, print_stdout_on_error=False)
if c1 == c2:
test_ret = c2
break
gitcp = None
if test_ret:
print('git bisect good')
gitcp = subprocess.run('git bisect good',
shell=True,
capture_output=True,
text=True)
else:
print('git bisect bad')
gitcp = subprocess.run('git bisect bad',
shell=True,
capture_output=True,
text=True)
# git should always print 'left to test after this'. No stdout
# means something is wrong.
if not gitcp.stdout:
print('Something is wrong! Exit bisect.')
if gitcp.stderr:
print(gitcp.stderr)
break
print(gitcp.stdout)
first_line = gitcp.stdout[:gitcp.stdout.find('\n')]
# Found the culprit!
if GIT_BAD_COMMENT_MSG in first_line:
print('Found the culprit change!')
return 0
if GIT_BISECT_IN_PROCESS_MSG not in first_line:
print('Something is wrong! Exit bisect.')
if gitcp.stderr:
print(gitcp.stderr)
break
return 1
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-b',
'--bad',
type=str,
help='A bad revision to start bisection.')
parser.add_argument('-g',
'--good',
type=str,
help='A good revision to start bisection.')
parser.add_argument('--build_command',
type=str,
help='Command to build test target.')
parser.add_argument('--test_command', type=str, help='Command to run test.')
args = parser.parse_args()
return StartBisect(args.good, args.bad, args.build_command, args.test_command)
if __name__ == '__main__':
sys.exit(main())
Codebase Browser
chromium