#!/usr/bin/env vpython3
# Copyright (C) 2021 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import argparse
import contextlib
import json
import logging
import os
import sys
import tempfile
from blinkpy.common.host import Host
from blinkpy.common.system.executive import ScriptError
from blinkpy.web_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
from blinkpy.web_tests.port.android import (
PRODUCTS, PRODUCTS_TO_STEPNAMES)
CSV_HEADING = ('Test name, %s Result, %s Result, '
'Result Comparison, Test Flaky Results, '
'Baseline Flaky Results, Unreliable Comparison\n')
YES = 'Yes'
NO = 'No'
_log = logging.getLogger(os.path.basename(__file__))
# Extend this script to compare the results between wptrunner/Chrome
# and rwt/content_shell on Linux
PRODUCTS = PRODUCTS + [
'android_webview', 'chrome_android', 'chrome_linux', 'content_shell',
'wpt_content_shell_linux', 'wpt_content_shell_win10',
'wpt_content_shell_win11', 'wpt_content_shell_mac11',
'wpt_content_shell_mac12', 'wpt_content_shell_mac12_arm64',
'wpt_content_shell_mac13', 'wpt_content_shell_mac13_arm64'
]
PRODUCTS_TO_STEPNAMES.update({
'android_webview': 'system_webview_wpt',
'chrome_android': 'chrome_public_wpt',
'chrome_linux': 'wpt_tests_suite',
'wpt_content_shell_linux': 'wpt_tests_suite_linux',
'wpt_content_shell_win10': 'wpt_tests_suite_win10',
'wpt_content_shell_win11': 'wpt_tests_suite_win11',
'wpt_content_shell_mac11': 'wpt_tests_suite_mac_11',
'wpt_content_shell_mac12': 'wpt_tests_suite_mac12',
'wpt_content_shell_mac12_arm64': 'wpt_tests_suite_mac12_arm64',
'wpt_content_shell_mac13': 'wpt_tests_suite_mac13',
'wpt_content_shell_mac13_arm64': 'wpt_tests_suite_mac13_arm64',
'content_shell': 'blink_wpt_tests'
})
PRODUCTS_TO_BUILDER_NAME = {
'android_webview': 'android-webview-pie-x86-wpt-fyi-rel',
'chrome_android': 'android-chrome-pie-x86-wpt-fyi-rel',
'chrome_linux': 'linux-wpt-chromium-rel',
'wpt_content_shell_linux': 'linux-wpt-content-shell-fyi-rel',
'wpt_content_shell_win10': 'win10-wpt-content-shell-fyi-rel',
'wpt_content_shell_win11': 'win11-wpt-content-shell-fyi-rel',
'wpt_content_shell_mac11': 'mac11-wpt-content-shell-fyi-rel',
'wpt_content_shell_mac12': 'mac12-wpt-content-shell-fyi-rel',
'wpt_content_shell_mac12_arm64': 'mac12-arm64-wpt-content-shell-fyi-rel',
'wpt_content_shell_mac13': 'mac13-wpt-content-shell-fyi-rel',
'wpt_content_shell_mac13_arm64': 'mac13-arm64-wpt-content-shell-fyi-rel',
'content_shell': "Linux Tests"
}
STEP_NAME_VARIANTS = {
'chrome_public_wpt': ['chrome_public_wpt on Ubuntu-16.04 or Ubuntu-18.04'],
'system_webview_wpt':
['system_webview_wpt on Ubuntu-16.04 or Ubuntu-18.04'],
'wpt_tests_suite_linux':
['wpt_tests_suite (experimental) on Ubuntu-18.04'],
'wpt_tests_suite_win10':
['wpt_tests_suite (experimental) on Windows-10-19042'],
'wpt_tests_suite_win11':
['wpt_tests_suite (experimental) on Windows-11-22000'],
'wpt_tests_suite_mac12': ['wpt_tests_suite (experimental) on Mac-12'],
'wpt_tests_suite_mac11': ['wpt_tests_suite (experimental) on Mac-11'],
'wpt_tests_suite_mac12_arm64':
['wpt_tests_suite (experimental) on Mac-12'],
'wpt_tests_suite_mac13': ['wpt_tests_suite (experimental) on Mac-13'],
'wpt_tests_suite_mac13_arm64':
['wpt_tests_suite (experimental) on Mac-13'],
'blink_wpt_tests': ['blink_wpt_tests on Ubuntu-18.04']
}
def map_tests_to_results(output_mp, input_mp, path=''):
if 'actual' in input_mp:
output_mp[path[1:]] = input_mp
else:
# TODO: needs to be extended when we support virtual tests
for k, v in input_mp.items():
if k == 'wpt_internal':
map_tests_to_results(output_mp, v, "/wpt_internal")
else:
map_tests_to_results(output_mp, v, path + '/' + k)
class WPTResultsDiffer(object):
def __init__(self, args, host, actual_results_map,
baseline_results_map, csv_output, ignore_missing=False):
self._args = args
self._host = host
self._actual_results_map = actual_results_map
self._baseline_results_map = baseline_results_map
self._csv_output = csv_output
self._ignore_missing = ignore_missing
self._test_flaky_results = None
self._baseline_flaky_results = None
try:
self._test_flaky_results = self._get_flaky_test_results(
args.product_to_compare)
except:
_log.info('Failed to get flaky results for %s' % args.product_to_compare)
try:
self._baseline_flaky_results = self._get_flaky_test_results(
args.baseline_product)
except:
_log.info('Failed to get flaky results for %s' % args.baseline_product)
self._handle_flaky = self._test_flaky_results is not None \
and self._baseline_flaky_results is not None
def _get_flaky_test_results(self, product):
return self._get_bot_expectations(product).flakes_by_path(
False, ignore_bot_expected_results=True,
consider_only_flaky_runs=False)
def _get_bot_expectations(self, product):
builder_name = PRODUCTS_TO_BUILDER_NAME[product]
bot_expectations_factory = BotTestExpectationsFactory(
self._host.builders, PRODUCTS_TO_STEPNAMES[product])
return bot_expectations_factory.expectations_for_builder(builder_name)
def flaky_results(self, test_name, flaky_dict):
return (flaky_dict.get(test_name, set()) or
flaky_dict.get(test_name.replace('external/wpt/', ''), set()))
def create_csv(self):
super_set = (set(self._actual_results_map.keys()) |
set(self._baseline_results_map.keys()))
file_output = CSV_HEADING % (self._args.product_to_compare,
self._args.baseline_product)
for test in sorted(super_set):
if ',' in test:
line = ['"%s"' % test]
else:
line = [test]
for result_mp in [self._actual_results_map,
self._baseline_results_map]:
line.append(result_mp.get(test, {'actual': 'MISSING'})
.get('actual').split()[-1])
if line[-1] == line[-2]:
line.append('SAME RESULTS')
elif 'MISSING' in (line[-1], line[-2]):
if self._ignore_missing:
continue
line.append('MISSING RESULTS')
else:
line.append('DIFFERENT RESULTS')
if self._handle_flaky and line[-1] != 'MISSING RESULTS':
test_flaky_results = self.flaky_results(
test, self._test_flaky_results)
baseline_flaky_results = self.flaky_results(
test, self._baseline_flaky_results)
test_flaky_results.update([line[1]])
baseline_flaky_results.update([line[2]])
is_flaky = (len(test_flaky_results) > 1 or
len(baseline_flaky_results) > 1)
line.extend(['"{%s}"' % ', '.join(sorted(test_flaky_results)),
'"{%s}"' % ', '.join(sorted(baseline_flaky_results))])
if (is_flaky and line[1] in baseline_flaky_results and
line[2] in test_flaky_results):
line.append(YES)
else:
line.append(NO)
else:
line.extend(['{}', '{}', NO])
file_output += ','.join(line) + '\n'
self._csv_output.write(file_output)
def _get_build_test_results(host, product, build):
step_name = PRODUCTS_TO_STEPNAMES[product]
for step_name_var in STEP_NAME_VARIANTS[step_name] + [step_name]:
try:
build_results = host.bb_agent.get_build_test_results(
build, step_name_var)
return build_results
except ScriptError:
_log.debug(('%s is not a step name that ran on %s:%d. '
'Re-attempting with a different step name.'),
step_name_var, build.builder_name, build.build_number)
except Exception as ex:
_log.exception(('Exception was raised when attempting to get '
'test results for step %s on build %s:%d'),
step_name_var, build.builder_name,
build.build_number)
raise ex
@contextlib.contextmanager
def _get_product_test_results(host, product, build_number, results_path=None):
if results_path:
json_results_obj = open(results_path, 'r')
else:
_log.info(('Retrieving test results for '
'product %s using the bb command'), product)
builder_name = PRODUCTS_TO_BUILDER_NAME[product]
# TODO: Note the builder name and number in the CSV file
build = host.bb_agent.get_finished_build(builder_name, build_number)
_log.debug('Using build %s(%d)', builder_name, build.build_number)
build_results = _get_build_test_results(host, product, build)
json_results_obj = tempfile.NamedTemporaryFile(mode='w+t')
json_results_obj.write(json.dumps(build_results))
json_results_obj.seek(0)
try:
yield json_results_obj
finally:
json_results_obj.close()
def main(args):
parser = argparse.ArgumentParser(prog=os.path.basename(__file__))
parser.add_argument('--baseline-test-results', required=False,
help='Path to baseline test results JSON file')
parser.add_argument('--baseline-product', required=True, action='store',
choices=PRODUCTS,
help='Name of the baseline product')
parser.add_argument(
'--baseline-build-number',
type=int,
default=0,
help='The baseline builder to fetch results, default to latest.')
parser.add_argument('--test-results-to-compare', required=False,
help='Path to actual test results JSON file')
parser.add_argument('--product-to-compare', required=True, action='store',
choices=PRODUCTS,
help='Name of the product being compared')
parser.add_argument(
'--compare-build-number',
type=int,
default=0,
help=
'The product to compare builder to fetch results, default to latest.')
parser.add_argument('--csv-output', required=True,
help='Path to CSV output file')
parser.add_argument('--verbose', '-v', action='count', default=1,
help='Verbosity level')
parser.add_argument('--ignore-missing', action='store_true',
required=False, default=False,
help='Ignore tests that are not run for one of the product')
args = parser.parse_args()
if args.verbose >= 3:
log_level = logging.DEBUG
elif args.verbose == 2:
log_level = logging.INFO
else:
log_level = logging.WARNING
logging.basicConfig(level=log_level)
assert args.product_to_compare != args.baseline_product, (
'Product to compare and the baseline product cannot be the same')
host = Host()
actual_results_getter = _get_product_test_results(
host, args.product_to_compare, args.compare_build_number,
args.test_results_to_compare)
baseline_results_getter = _get_product_test_results(
host, args.baseline_product, args.baseline_build_number,
args.baseline_test_results)
with actual_results_getter as actual_results_content, \
baseline_results_getter as baseline_results_content, \
open(args.csv_output, 'w') as csv_output:
# Read JSON results files. They must follow the Chromium
# json test results format
actual_results_json = json.loads(actual_results_content.read())
baseline_results_json = json.loads(baseline_results_content.read())
# Compress the JSON results trie in order to map the test
# names to their results map
tests_to_actual_results = {}
tests_to_baseline_results = {}
if (args.product_to_compare.startswith('chrome_linux')
or args.product_to_compare.startswith('wpt_content_shell')):
path = '/external/wpt'
else:
path = ''
map_tests_to_results(tests_to_actual_results,
actual_results_json['tests'],
path=path)
if (args.baseline_product.startswith('chrome_linux')
or args.baseline_product.startswith('wpt_content_shell')):
path = '/external/wpt'
else:
path = ''
map_tests_to_results(tests_to_baseline_results,
baseline_results_json['tests'],
path=path)
# Create a CSV file which compares tests results to baseline results
WPTResultsDiffer(args, host, tests_to_actual_results,
tests_to_baseline_results, csv_output,
args.ignore_missing).create_csv()
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
Codebase Browser
chromium