# Copyright 2013 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""web_tests/ presubmit script for Blink.
See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""
import filecmp
import inspect
import os
import sys
import tempfile
import re
from html.parser import HTMLParser
from typing import List
def _CheckTestharnessWdspecResults(input_api, output_api):
"""Checks for all-PASS generic baselines for testharness/wdspec tests.
These files are unnecessary because for testharness/wdspec tests, if there is no
baseline file then the test is considered to pass when the output is all
PASS. Note that only generic baselines are checked because platform specific
and virtual baselines might be needed to prevent fallback.
"""
baseline_files = _TxtGenericBaselinesToCheck(input_api)
if not baseline_files:
return []
checker_path = input_api.os_path.join(input_api.PresubmitLocalPath(), '..',
'tools', 'check_expected_pass.py')
# When running git cl presubmit --all this presubmit may be asked to check
# ~19,000 files. Passing these on the command line would far exceed Windows
# limits, so we use --path-files instead.
# We have to set delete=False and then let the object go out of scope so
# that the file can be opened by name on Windows.
with tempfile.NamedTemporaryFile('w+', newline='', delete=False) as f:
for path in baseline_files:
f.write('%s\n' % path)
paths_name = f.name
args = [
input_api.python3_executable, checker_path, '--path-files', paths_name
]
_, errs = input_api.subprocess.Popen(
args,
stdout=input_api.subprocess.PIPE,
stderr=input_api.subprocess.PIPE,
universal_newlines=True).communicate()
os.remove(paths_name)
if errs:
return [output_api.PresubmitError(errs)]
return []
def _TxtGenericBaselinesToCheck(input_api):
"""Returns a list of paths of generic baselines for testharness/wdspec tests."""
baseline_files = []
this_dir = input_api.PresubmitLocalPath()
for f in input_api.AffectedFiles():
if f.Action() == 'D':
continue
path = f.AbsoluteLocalPath()
if not path.endswith('-expected.txt'):
continue
if (input_api.os_path.join(this_dir, 'platform') in path
or input_api.os_path.join(this_dir, 'virtual') in path
or input_api.os_path.join(this_dir, 'flag-specific') in path):
continue
baseline_files.append(path)
return baseline_files
def _CheckFilesUsingEventSender(input_api, output_api):
"""Check if any new layout tests still use eventSender. If they do, we encourage replacing them with
chrome.gpuBenchmarking.pointerActionSequence.
"""
results = []
actions = ["eventSender.touch", "eventSender.mouse", "eventSender.gesture"]
for f in input_api.AffectedFiles():
if f.Action() == 'A':
for line_num, line in f.ChangedContents():
if line.find("eventSender.beginDragWithFiles") != -1:
break
if any(action in line for action in actions):
results.append(output_api.PresubmitPromptWarning(
'eventSender is deprecated, please use chrome.gpuBenchmarking.pointerActionSequence instead ' +
'(see https://crbug.com/711340 and http://goo.gl/BND75q).\n' +
'Files: %s:%d %s ' % (f.LocalPath(), line_num, line)))
return results
def _CheckTestExpectations(input_api, output_api):
results = []
os_path = input_api.os_path
sys.path.append(
os_path.join(
os_path.dirname(
os_path.abspath(inspect.getfile(_CheckTestExpectations))),
'..', 'tools'))
from blinkpy.presubmit.lint_test_expectations import (
PresubmitCheckTestExpectations)
results.extend(PresubmitCheckTestExpectations(input_api, output_api))
return results
def _CheckForRedundantBaselines(input_api, output_api, max_tests: int = 1000):
tests = _TestsCorrespondingToAffectedBaselines(input_api, max_tests)
if not tests:
return []
elif len(tests) > max_tests:
return [
output_api.PresubmitNotifyResult(
'Too many tests to check for redundant baselines; skipping.',
items=tests),
]
path_to_blink_tool = input_api.os_path.join(input_api.PresubmitLocalPath(),
input_api.os_path.pardir,
'tools', 'blink_tool.py')
with input_api.CreateTemporaryFile(mode='w+') as test_name_file:
for test in tests:
test_name_file.write(f'{test}\n')
test_name_file.flush()
command_args = [
input_api.python3_executable,
path_to_blink_tool,
'optimize-baselines',
'--no-manifest-update',
'--check',
f'--test-name-file={test_name_file.name}',
]
command = input_api.Command(
name='Checking for redundant affected baselines ...',
cmd=command_args,
kwargs={},
message=output_api.PresubmitPromptWarning,
python3=True)
return input_api.RunTests([command])
def _TestsCorrespondingToAffectedBaselines(input_api,
max_tests: int = 1000) -> List[str]:
sep = input_api.re.escape(input_api.os_path.sep)
baseline_pattern = input_api.re.compile(
r'((platform|flag-specific)%s[^%s]+%s)?(virtual%s[^%s]+%s)?'
r'(?P<test_prefix>.*)-expected\.(txt|png|wav)' % ((sep, ) * 6))
test_paths = set()
for affected_file in input_api.AffectedFiles():
if len(test_paths) > max_tests:
# Exit early; no need to glob for more tests.
break
baseline_path_from_web_tests = input_api.os_path.relpath(
affected_file.AbsoluteLocalPath(), input_api.PresubmitLocalPath())
baseline_match = baseline_pattern.fullmatch(
baseline_path_from_web_tests)
if not baseline_match:
continue
# Baselines for WPT-style variants have sanitized filenames with '?' and
# '&' in the query parameter section converted to '_', like:
# a/b.html?c&d -> a/b_c_d-expected.txt
#
# Regrettably, this is a lossy coercion that cannot be distinguished
# from tests with '_' in the original test ID, like:
# a/b_c_d.html -> a/b_c_d-expected.txt
#
# Here, we err on the side of not attempting to check such tests, which
# could be unrelated.
test_prefix = baseline_match['test_prefix']
# Getting the test name from the baseline path is not as easy as the
# other direction. Try all extensions as a heuristic instead.
for extension in [
'html', 'xml', 'xhtml', 'xht', 'pl', 'htm', 'php', 'svg',
'mht', 'pdf', 'js'
]:
abs_prefix = input_api.os_path.join(input_api.PresubmitLocalPath(),
test_prefix)
test_paths.update(input_api.glob(f'{abs_prefix}*.{extension}'))
return [
input_api.os_path.relpath(test_path, input_api.PresubmitLocalPath())
for test_path in sorted(test_paths)
]
def _CheckForJSTest(input_api, output_api):
"""'js-test.js' is the past, 'testharness.js' is our glorious future"""
jstest_re = input_api.re.compile(r'resources/js-test.js')
def source_file_filter(path):
return input_api.FilterSourceFile(path, files_to_check=[r'\.(html|js|php|pl|svg)$'])
errors = input_api.canned_checks._FindNewViolationsOfRule(
lambda _, x: not jstest_re.search(x), input_api, source_file_filter)
errors = [' * %s' % violation for violation in errors]
if errors:
return [output_api.PresubmitPromptOrNotify(
'"resources/js-test.js" is deprecated; please write new layout '
'tests using the assertions in "resources/testharness.js" '
'instead, as these can be more easily upstreamed to Web Platform '
'Tests for cross-vendor compatibility testing. If you\'re not '
'already familiar with this framework, a tutorial is available at '
'https://darobin.github.io/test-harness-tutorial/docs/using-testharness.html'
'\n\n%s' % '\n'.join(errors))]
return []
def _CheckForInvalidPreferenceError(input_api, output_api):
pattern = input_api.re.compile('Invalid name for preference: (.+)')
results = []
for f in input_api.AffectedFiles():
if not f.LocalPath().endswith('-expected.txt'):
continue
for line_num, line in f.ChangedContents():
error = pattern.search(line)
if error:
results.append(output_api.PresubmitError('Found an invalid preference %s in expected result %s:%s' % (error.group(1), f, line_num)))
return results
def _CheckRunAfterLayoutAndPaintJS(input_api, output_api):
"""Checks if resources/run-after-layout-and-paint.js and
http/tests/resources/run-after-layout-and-paint.js are the same."""
js_file = input_api.os_path.join(input_api.PresubmitLocalPath(),
'resources', 'run-after-layout-and-paint.js')
http_tests_js_file = input_api.os_path.join(input_api.PresubmitLocalPath(),
'http', 'tests', 'resources', 'run-after-layout-and-paint.js')
for f in input_api.AffectedFiles():
path = f.AbsoluteLocalPath()
if path == js_file or path == http_tests_js_file:
if not filecmp.cmp(js_file, http_tests_js_file):
return [output_api.PresubmitError(
'%s and %s must be kept exactly the same' %
(js_file, http_tests_js_file))]
break
return []
def _CheckForUnlistedTestFolder(input_api, output_api):
"""Checks all the test folders under web_tests are listed in BUILD.gn.
"""
this_dir = input_api.PresubmitLocalPath()
possible_new_dirs = set()
for f in input_api.AffectedFiles():
if f.Action() == 'A':
# We only check added folders. For deleted folders, if BUILD.gn is
# not updated, the build will fail at upload step. The reason is that
# we can not know if the folder is deleted as there can be local
# unchecked in files.
path = f.AbsoluteLocalPath()
fns = path[len(this_dir)+1:].split('/')
if len(fns) > 1:
possible_new_dirs.add(fns[0])
if possible_new_dirs:
path_build_gn = input_api.os_path.join(input_api.change.RepositoryRoot(), 'BUILD.gn')
dirs_from_build_gn = []
start_line = '# === List Test Cases folders here ==='
end_line = '# === Test Case Folders Ends ==='
end_line_count = 0
find_start_line = False
for line in input_api.ReadFile(path_build_gn).splitlines():
line = line.strip()
if line.startswith(start_line):
find_start_line = True
continue
if find_start_line:
if line.startswith(end_line):
find_start_line = False
end_line_count += 1
if end_line_count == 2:
break
continue
if len(line.split('/')) > 1:
dirs_from_build_gn.append(line.split('/')[-2])
dirs_from_build_gn.extend(
['platform', 'FlagExpectations', 'flag-specific', 'TestLists'])
new_dirs = [x for x in possible_new_dirs if x not in dirs_from_build_gn]
if new_dirs:
dir_plural = "directories" if len(new_dirs) > 1 else "directory"
error_message = (
'This CL adds new %s(%s) under //third_party/blink/web_tests/, but //BUILD.gn '
'is not updated. Please add the %s to BUILD.gn.' % (dir_plural, ', '.join(new_dirs), dir_plural))
if input_api.is_committing:
return [output_api.PresubmitError(error_message)]
else:
return [output_api.PresubmitPromptWarning(error_message)]
return []
def _CheckForExtraVirtualBaselines(input_api, output_api):
"""Checks that expectations in virtual test suites are for virtual test suites that exist
"""
# This test fails on Windows because win32pipe is not available and
# other errors.
if os.name == 'nt':
return []
os_path = input_api.os_path
local_dir = os_path.relpath(
os_path.normpath('{0}/'.format(input_api.PresubmitLocalPath().replace(
os_path.sep, '/'))), input_api.change.RepositoryRoot())
check_all = False
check_files = []
for f in input_api.AffectedFiles(include_deletes=False):
local_path = f.LocalPath()
assert local_path.startswith(local_dir)
local_path = os_path.relpath(local_path, local_dir)
path_components = local_path.split(os_path.sep)
if f.Action() == 'A':
if len(path_components) > 2 and path_components[0] == 'virtual':
check_files.append((local_path, path_components[1]))
elif (len(path_components) > 4 and path_components[2] == 'virtual'
and (path_components[0] == 'platform'
or path_components[0] == 'flag-specific')):
check_files.append((local_path, path_components[3]))
elif local_path == 'VirtualTestSuites':
check_all = True
if not check_all and len(check_files) == 0:
return []
from blinkpy.common.host import Host
port_factory = Host().port_factory
known_virtual_suites = [
suite.full_prefix[8:-1] for suite in port_factory.get(
port_factory.all_port_names()[0]).virtual_test_suites()
]
results = []
if check_all:
for f in input_api.change.AllFiles(
os_path.join(input_api.PresubmitLocalPath(), "virtual")):
suite = f.split('/')[0]
if not suite in known_virtual_suites:
path = os_path.relpath(
os_path.join(input_api.PresubmitLocalPath(), "virtual", f),
input_api.change.RepositoryRoot())
results.append(
output_api.PresubmitError(
"Baseline %s exists, but %s is not a known virtual test suite."
% (path, suite)))
for subdir in ["platform", "flag-specific"]:
for f in input_api.change.AllFiles(
os_path.join(input_api.PresubmitLocalPath(), subdir)):
path_components = f.split('/')
if len(path_components) < 3 or path_components[1] != 'virtual':
continue
suite = path_components[2]
if not suite in known_virtual_suites:
path = os_path.relpath(
os_path.join(input_api.PresubmitLocalPath(), subdir,
f), input_api.change.RepositoryRoot())
results.append(
output_api.PresubmitError(
"Baseline %s exists, but %s is not a known virtual test suite."
% (path, suite)))
else:
for (f, suite) in check_files:
if not suite in known_virtual_suites:
path = os_path.relpath(
os_path.join(input_api.PresubmitLocalPath(), f),
input_api.change.RepositoryRoot())
results.append(
output_api.PresubmitError(
"This CL adds a new baseline %s, but %s is not a known virtual test suite."
% (path, suite)))
return results
def _CheckWebViewExpectations(input_api, output_api):
src_dir = os.path.join(input_api.PresubmitLocalPath(), os.pardir,
os.pardir, os.pardir)
webview_data_dir = input_api.os_path.join(src_dir, 'android_webview',
'tools', 'system_webview_shell',
'test', 'data', 'webexposed')
if webview_data_dir not in sys.path:
sys.path.append(webview_data_dir)
# pylint: disable=import-outside-toplevel
from exposed_webview_interfaces_presubmit import (
CheckNotWebViewExposedInterfaces)
return CheckNotWebViewExposedInterfaces(input_api, output_api)
class _DoctypeParser(HTMLParser):
"""Parses HTML to check if there exists a DOCTYPE declaration before all other tags.
"""
def __init__(self):
super().__init__()
self.encountered_tag = False
self.doctype = ""
def handle_starttag(self, *_):
self.encountered_tag = True
def handle_startendtag(self, *_):
self.encountered_tag = True
def handle_decl(self, decl):
if not self.encountered_tag:
self.doctype = decl
self.encountered_tag = True
def _IsDoctypeHTMLSet(lines):
"""Returns true if the given HTML file starts with <!DOCTYPE html>.
"""
parser = _DoctypeParser()
for l in lines:
parser.feed(l)
return re.match("DOCTYPE\s*html\s*$", parser.doctype, re.IGNORECASE)
def _CheckForDoctypeHTML(input_api, output_api):
"""Checks that all changed HTML files start with the correct <!DOCTYPE html> tag.
"""
results = []
if input_api.no_diffs:
return results
wpt_path = input_api.os_path.join(input_api.PresubmitLocalPath(),
"external", "wpt")
for f in input_api.AffectedFiles(include_deletes=False):
path = f.LocalPath()
fname = input_api.os_path.basename(path)
if not fname.endswith(".html") or "quirk" in fname:
continue
if not _IsDoctypeHTMLSet(f.NewContents()):
error = "HTML file \"%s\" does not start with <!DOCTYPE html>. " \
"If you really intend to test in quirks mode, add \"quirk\" " \
"to the name of your test." % path
if f.Action() == "A" or _IsDoctypeHTMLSet(f.OldContents()):
# These tests are being imported from WPT, so <!DOCTYPE html> is
# not required yet.
no_errors = f.AbsoluteLocalPath().startswith(wpt_path)
if no_errors:
results.append(output_api.PresubmitPromptWarning(error))
else:
results.append(output_api.PresubmitError(error))
return results
def _CheckNewVirtualSuites(input_api, output_api, max_suite_length: int = 48):
"""Validate new virtual test suites."""
# TODO(crbug.com/1380165): Once all virtual suites adopt "owners", consider
# making the field mandatory. In that case, we don't need to access the
# change contents and can promote this check to `lint_test_expectations.py`.
vts_path = input_api.os_path.join(input_api.PresubmitLocalPath(),
'VirtualTestSuites')
results = []
for affected_file in input_api.AffectedFiles():
if affected_file.AbsoluteLocalPath() != vts_path:
continue
old_contents = ''.join(affected_file.OldContents())
new_contents = ''.join(affected_file.NewContents())
try:
old_suites = _FilterForSuites(input_api.json.loads(old_contents))
new_suites = _FilterForSuites(input_api.json.loads(new_contents))
old_suite_names = {suite['prefix'] for suite in old_suites}
new_ownerless_suites, new_long_suites = [], []
for suite in new_suites:
prefix, owners = suite['prefix'], suite.get('owners', [])
if prefix in old_suite_names:
continue
if not owners:
new_ownerless_suites.append(prefix)
if len(prefix) > max_suite_length:
new_long_suites.append(prefix)
if new_ownerless_suites:
results.append(
output_api.PresubmitPromptWarning(
'Consider specifying "owners" (a list of emails) '
'for the virtual suites added by this patch:',
new_ownerless_suites))
if new_long_suites:
results.append(
output_api.PresubmitPromptWarning(
'Consider shorter virtual suite names so that the '
"global filename length presubmit doesn't reject "
'future `*-expected.txt` under their directories. You '
'can add comments about these suites to '
'VirtualTestSuites.', new_long_suites))
except (ValueError, KeyError):
# Invalid JSON or missing required fields will be detected by
# `lint_test_expectations.py`.
pass
break
return results
def _FilterForSuites(suites):
return [suite for suite in suites if not isinstance(suite, str)]
def CheckChangeOnUpload(input_api, output_api):
results = []
results.extend(_CheckTestharnessWdspecResults(input_api, output_api))
results.extend(_CheckFilesUsingEventSender(input_api, output_api))
results.extend(_CheckTestExpectations(input_api, output_api))
# `_CheckTestExpectations()` updates the WPT manifests for
# `_CheckForRedundantBaselines()`, so they must run in order. (Updating the
# manifest is needed to correctly detect tests but takes 10-15s, so try
# to only do so once; see crbug.com/1492238.)
results.extend(_CheckForRedundantBaselines(input_api, output_api))
results.extend(_CheckForJSTest(input_api, output_api))
results.extend(_CheckForInvalidPreferenceError(input_api, output_api))
results.extend(_CheckRunAfterLayoutAndPaintJS(input_api, output_api))
results.extend(_CheckForUnlistedTestFolder(input_api, output_api))
results.extend(_CheckForExtraVirtualBaselines(input_api, output_api))
results.extend(_CheckWebViewExpectations(input_api, output_api))
results.extend(_CheckForDoctypeHTML(input_api, output_api))
results.extend(_CheckNewVirtualSuites(input_api, output_api))
return results
def CheckChangeOnCommit(input_api, output_api):
results = []
results.extend(_CheckTestharnessWdspecResults(input_api, output_api))
results.extend(_CheckFilesUsingEventSender(input_api, output_api))
results.extend(_CheckTestExpectations(input_api, output_api))
# `_CheckTestExpectations()` updates the WPT manifests for
# `_CheckForRedundantBaselines()`, so they must run in order. (Updating the
# manifest is needed to correctly detect tests but takes 10-15s, so try
# to only do so once; see crbug.com/1492238.)
results.extend(_CheckForRedundantBaselines(input_api, output_api))
results.extend(_CheckForUnlistedTestFolder(input_api, output_api))
results.extend(_CheckForExtraVirtualBaselines(input_api, output_api))
results.extend(_CheckWebViewExpectations(input_api, output_api))
results.extend(_CheckForDoctypeHTML(input_api, output_api))
results.extend(_CheckNewVirtualSuites(input_api, output_api))
return results
Codebase Browser
chromium