#!/usr/bin/env vpython3
# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import os
import tempfile
import unittest
from pyfakefs import fake_filesystem_unittest
import test_filtering
from test_filtering import _TestFilter
from test_filtering import _TestFiltersGroup
from test_filtering import _SetOfTestFiltersGroups
# Protected access is allowed for unittests.
# pylint: disable=protected-access
class FilterTests(fake_filesystem_unittest.TestCase):
def test_exact_match(self):
t = _TestFilter('foo')
self.assertTrue(t.is_match('foo'))
self.assertFalse(t.is_match('foobar'))
self.assertFalse(t.is_match('foo/bar'))
self.assertFalse(t.is_match('fo'))
self.assertFalse(t.is_exclusion_filter())
def test_prefix_match(self):
t = _TestFilter('foo*')
self.assertTrue(t.is_match('foo'))
self.assertTrue(t.is_match('foobar'))
self.assertTrue(t.is_match('foo/bar'))
self.assertFalse(t.is_match('fo'))
self.assertFalse(t.is_exclusion_filter())
def test_exclusion_match(self):
t = _TestFilter('-foo')
self.assertTrue(t.is_match('foo'))
self.assertFalse(t.is_match('foobar'))
self.assertFalse(t.is_match('foo/bar'))
self.assertFalse(t.is_match('fo'))
self.assertTrue(t.is_exclusion_filter())
def test_exclusion_prefix_match(self):
t = _TestFilter('-foo*')
self.assertTrue(t.is_match('foo'))
self.assertTrue(t.is_match('foobar'))
self.assertTrue(t.is_match('foo/bar'))
self.assertFalse(t.is_match('fo'))
self.assertTrue(t.is_exclusion_filter())
def test_error_conditions(self):
# '*' is only supported at the end
with self.assertRaises(ValueError):
_TestFilter('*.bar')
def _create_group_from_pseudo_file(file_contents):
# pylint: disable=unexpected-keyword-arg
with tempfile.NamedTemporaryFile(delete=False, mode='w',
encoding='utf-8') as f:
filepath = f.name
f.write(file_contents)
try:
return _TestFiltersGroup.from_filter_file(filepath)
finally:
os.remove(filepath)
class FiltersGroupTests(fake_filesystem_unittest.TestCase):
def test_single_positive_filter(self):
g = _TestFiltersGroup.from_string('foo*')
self.assertTrue(g.is_test_name_included('foo'))
self.assertTrue(g.is_test_name_included('foobar'))
self.assertFalse(g.is_test_name_included('baz'))
self.assertFalse(g.is_test_name_included('fo'))
def test_single_negative_filter(self):
g = _TestFiltersGroup.from_string('-foo*')
self.assertFalse(g.is_test_name_included('foo'))
self.assertFalse(g.is_test_name_included('foobar'))
self.assertTrue(g.is_test_name_included('baz'))
self.assertTrue(g.is_test_name_included('fo'))
def test_specificity_ordering(self):
# From test_executable_api.md#filtering-which-tests-to-run:
#
# If multiple filters in a flag match a given test name, the longest
# match takes priority (longest match wins). I.e.,. if you had
# --isolated-script-test-filter='a*::-ab*', then ace.html would run
# but abd.html would not. The order of the filters should not
# matter.
g1 = _TestFiltersGroup.from_string('a*::-ab*') # order #1
g2 = _TestFiltersGroup.from_string('-ab*::a*') # order #2
self.assertTrue(g1.is_test_name_included('ace'))
self.assertTrue(g2.is_test_name_included('ace'))
self.assertFalse(g1.is_test_name_included('abd'))
self.assertFalse(g2.is_test_name_included('abd'))
def test_specificity_conflicts(self):
# Docs give this specific example: It is an error to have multiple
# expressions of the same length that conflict (e.g., a*::-a*).
with self.assertRaises(ValueError):
_TestFiltersGroup.from_string('a*::-a*')
# Other similar conflict:
with self.assertRaises(ValueError):
_TestFiltersGroup.from_string('a::-a')
# It doesn't really make sense to support `foo.bar` and `foo.bar*` and
# have the latter take precedence over the former.
with self.assertRaises(ValueError):
_TestFiltersGroup.from_string('foo.bar::foo.bar*')
# In the same spirit, identical duplicates are also treated as
# conflicts.
with self.assertRaises(ValueError):
_TestFiltersGroup.from_string('foo.bar::foo.bar')
# Ok - no conflicts:
_TestFiltersGroup.from_string('a*::-b*') # Different filter text
def test_simple_list(self):
# 'simple test list' format from bit.ly/chromium-test-list-format
# (aka go/test-list-format)
file_content = """
# Comment
aaa
bbb # End-of-line comment
Bug(intentional) ccc [ Crash ] # Comment
crbug.com/12345 [ Debug] ddd
skbug.com/12345 [ Debug] eee
webkit.org/12345 [ Debug] fff
ggg*
-ggghhh
""".strip()
g = _create_group_from_pseudo_file(file_content)
self.assertTrue(g.is_test_name_included('aaa'))
self.assertTrue(g.is_test_name_included('bbb'))
self.assertTrue(g.is_test_name_included('ccc'))
self.assertTrue(g.is_test_name_included('ddd'))
self.assertFalse(g.is_test_name_included('aa'))
self.assertFalse(g.is_test_name_included('aaax'))
self.assertTrue(g.is_test_name_included('ggg'))
self.assertTrue(g.is_test_name_included('gggg'))
self.assertFalse(g.is_test_name_included('gg'))
self.assertFalse(g.is_test_name_included('ggghhh'))
self.assertFalse(g.is_test_name_included('zzz'))
def test_tagged_list(self):
# tagged list format from bit.ly/chromium-test-list-format
# (aka go/test-list-format)
file_content = """
# Comment
abc
foo* # End-of-line comment
-foobar*
""".strip()
g = _create_group_from_pseudo_file(file_content)
self.assertTrue(g.is_test_name_included('abc'))
self.assertFalse(g.is_test_name_included('abcd'))
self.assertTrue(g.is_test_name_included('foo'))
self.assertTrue(g.is_test_name_included('food'))
self.assertFalse(g.is_test_name_included('foobar'))
self.assertFalse(g.is_test_name_included('foobarbaz'))
class SetOfFilterGroupsTests(fake_filesystem_unittest.TestCase):
@classmethod
def setUpClass(cls):
# `_filter1`, `_filter2`, and `_tests` are based on the setup described
# in test_executable_api.md#examples
cls._filter1 = _TestFiltersGroup.from_string(
'Foo.Bar.*::-Foo.Bar.bar3')
cls._filter2 = _TestFiltersGroup.from_string('Foo.Bar.bar2')
cls._tests = [
'Foo.Bar.bar1',
'Foo.Bar.bar2',
'Foo.Bar.bar3',
'Foo.Baz.baz', # TODO: Fix typo in test_executable_api.md
'Foo.Quux.quux'
]
def test_basics(self):
# This test corresponds to
# test_executable_api.md#filtering-tests-on-the-command-line
# and
# test_executable_api.md#using-a-filter-file
s = _SetOfTestFiltersGroups([self._filter1])
self.assertEqual(['Foo.Bar.bar1', 'Foo.Bar.bar2'],
s.filter_test_names(self._tests))
def test_combining_multiple_filters1(self):
# This test corresponds to the first test under
# test_executable_api.md#combining-multiple-filters
s = _SetOfTestFiltersGroups([
_TestFiltersGroup.from_string('Foo.Bar.*'),
_TestFiltersGroup.from_string('Foo.Bar.bar2')
])
self.assertEqual(['Foo.Bar.bar2'], s.filter_test_names(self._tests))
def test_combining_multiple_filters2(self):
# This test corresponds to the second test under
# test_executable_api.md#combining-multiple-filters
# TODO([email protected]): Figure out if the 3rd test example from
# the docs has correct inputs+outputs (or if there are some typos).
s = _SetOfTestFiltersGroups([
_TestFiltersGroup.from_string('Foo.Bar.*'),
_TestFiltersGroup.from_string('Foo.Baz.baz')
])
self.assertEqual([], s.filter_test_names(self._tests))
class PublicApiTests(fake_filesystem_unittest.TestCase):
def test_filter_cmdline_arg(self):
parser = argparse.ArgumentParser()
test_filtering.add_cmdline_args(parser)
args = parser.parse_args(args=[
'--isolated-script-test-filter=-barbaz',
'--isolated-script-test-filter=foo*::bar*'
])
self.assertEqual(
['foo1', 'foo2', 'bar1', 'bar2'],
test_filtering.filter_tests(
args, {}, ['foo1', 'foo2', 'bar1', 'bar2', 'barbaz', 'zzz']))
def test_filter_file_cmdline_arg(self):
# pylint: disable=unexpected-keyword-arg
f = tempfile.NamedTemporaryFile(delete=False,
mode='w',
encoding='utf-8')
try:
filepath = f.name
f.write('foo*')
f.close()
parser = argparse.ArgumentParser()
test_filtering.add_cmdline_args(parser)
args = parser.parse_args(args=[
'--isolated-script-test-filter-file={0:s}'.format(filepath)
])
self.assertEqual(['foo1', 'foo2'],
test_filtering.filter_tests(
args, {}, ['foo1', 'foo2', 'bar1', 'bar2']))
finally:
os.remove(filepath)
def _shard_tests(input_test_list_string, input_env):
input_test_list = input_test_list_string.split(',')
output_test_list = test_filtering._shard_tests(input_test_list, input_env)
return ','.join(output_test_list)
class ShardingTest(unittest.TestCase):
def test_empty_environment(self):
self.assertEqual('a,b,c', _shard_tests('a,b,c', {}))
def test_basic_sharding(self):
self.assertEqual(
'a,c,e',
_shard_tests('a,b,c,d,e', {
'GTEST_SHARD_INDEX': '0',
'GTEST_TOTAL_SHARDS': '2'
}))
self.assertEqual(
'b,d',
_shard_tests('a,b,c,d,e', {
'GTEST_SHARD_INDEX': '1',
'GTEST_TOTAL_SHARDS': '2'
}))
def test_error_conditions(self):
# shard index > total shards
with self.assertRaises(Exception):
_shard_tests('', {
'GTEST_SHARD_INDEX': '2',
'GTEST_TOTAL_SHARDS': '2'
})
# non-integer shard index
with self.assertRaises(Exception):
_shard_tests('', {
'GTEST_SHARD_INDEX': 'a',
'GTEST_TOTAL_SHARDS': '2'
})
# non-integer total shards
with self.assertRaises(Exception):
_shard_tests('', {
'GTEST_SHARD_INDEX': '0',
'GTEST_TOTAL_SHARDS': 'b'
})
if __name__ == '__main__':
unittest.main()
Codebase Browser
chromium