#!/usr/bin/env vpython3
# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# pylint: disable=protected-access
import datetime
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pyfakefs import fake_filesystem_unittest # pylint:disable=import-error
from flake_suppressor_common import common_typing as ct
from flake_suppressor_common import expectations
from flake_suppressor_common import unittest_utils as uu
# Note for all tests in this class: We can safely check the contents of the file
# at the end despite potentially having multiple added lines because Python 3.7+
# guarantees that dictionaries remember insertion order, so there is no risk of
# the order of modification changing.
@unittest.skipIf(sys.version_info[0] != 3, 'Python 3-only')
class IterateThroughResultsForUserUnittest(fake_filesystem_unittest.TestCase):
def setUp(self) -> None:
self._new_stdout = open(os.devnull, 'w')
self.setUpPyfakefs()
self._expectations = uu.UnitTestExpectationProcessor()
# Redirect stdout since the tested function prints a lot.
self._old_stdout = sys.stdout
sys.stdout = self._new_stdout
self._input_patcher = mock.patch.object(expectations.ExpectationProcessor,
'PromptUserForExpectationAction')
self._input_mock = self._input_patcher.start()
self.addCleanup(self._input_patcher.stop)
self.result_map = {
'pixel_integration_test': {
'foo_test': {
tuple(['win']): ['a'],
tuple(['mac']): ['b'],
},
'bar_test': {
tuple(['win']): ['c'],
},
},
}
self.expectation_file = os.path.join(uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY,
'pixel_expectations.txt')
uu.CreateFile(self, self.expectation_file)
expectation_file_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
self._expectation_file_patcher = mock.patch.object(
uu.UnitTestExpectationProcessor, 'GetExpectationFileForSuite')
self._expectation_file_mock = self._expectation_file_patcher.start()
self._expectation_file_mock.return_value = self.expectation_file
self.addCleanup(self._expectation_file_patcher.stop)
def tearDown(self) -> None:
sys.stdout = self._old_stdout
self._new_stdout.close()
def testIterateThroughResultsForUserIgnoreNoGroupByTags(self) -> None:
"""Tests that everything appears to function with ignore and no group."""
self._input_mock.return_value = (None, None)
self._expectations.IterateThroughResultsForUser(self.result_map, False,
True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testIterateThroughResultsForUserIgnoreGroupByTags(self) -> None:
"""Tests that everything appears to function with ignore and grouping."""
self._input_mock.return_value = (None, None)
self._expectations.IterateThroughResultsForUser(self.result_map, True, True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testIterateThroughResultsForUserRetryNoGroupByTags(self) -> None:
"""Tests that everything appears to function with retry and no group."""
self._input_mock.return_value = ('RetryOnFailure', '')
self._expectations.IterateThroughResultsForUser(self.result_map, False,
True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
[ win ] foo_test [ RetryOnFailure ]
[ mac ] foo_test [ RetryOnFailure ]
[ win ] bar_test [ RetryOnFailure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testIterateThroughResultsForUserRetryGroupByTags(self) -> None:
"""Tests that everything appears to function with retry and grouping."""
self._input_mock.return_value = ('RetryOnFailure', 'crbug.com/1')
self._expectations.IterateThroughResultsForUser(self.result_map, True, True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
crbug.com/1 [ win ] foo_test [ RetryOnFailure ]
crbug.com/1 [ win ] bar_test [ RetryOnFailure ]
[ mac ] some_test [ Failure ]
crbug.com/1 [ mac ] foo_test [ RetryOnFailure ]
[ android ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testIterateThroughResultsForUserFailNoGroupByTags(self) -> None:
"""Tests that everything appears to function with failure and no group."""
self._input_mock.return_value = ('Failure', 'crbug.com/1')
self._expectations.IterateThroughResultsForUser(self.result_map, False,
True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
crbug.com/1 [ win ] foo_test [ Failure ]
crbug.com/1 [ mac ] foo_test [ Failure ]
crbug.com/1 [ win ] bar_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testIterateThroughResultsForUserFailGroupByTags(self) -> None:
"""Tests that everything appears to function with failure and grouping."""
self._input_mock.return_value = ('Failure', '')
self._expectations.IterateThroughResultsForUser(self.result_map, True, True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ win ] foo_test [ Failure ]
[ win ] bar_test [ Failure ]
[ mac ] some_test [ Failure ]
[ mac ] foo_test [ Failure ]
[ android ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testIterateThroughResultsForUserNoIncludeAllTags(self) -> None:
"""Tests that everything appears to function without including all tags"""
self.result_map = {
'pixel_integration_test': {
'foo_test': {
tuple(['win', 'win10']): ['a'],
tuple(['mac']): ['b'],
},
'bar_test': {
tuple(['win']): ['c'],
},
},
}
self._input_mock.return_value = ('RetryOnFailure', '')
self._expectations.IterateThroughResultsForUser(self.result_map, False,
False)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
[ win10 ] foo_test [ RetryOnFailure ]
[ mac ] foo_test [ RetryOnFailure ]
[ win ] bar_test [ RetryOnFailure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
@unittest.skipIf(sys.version_info[0] != 3, 'Python 3-only')
class IterateThroughResultsWithThresholdsUnittest(
fake_filesystem_unittest.TestCase):
def setUp(self) -> None:
self.setUpPyfakefs()
self._expectations = uu.UnitTestExpectationProcessor()
self.result_map = {
'pixel_integration_test': {
'foo_test': {
tuple(['win']): ['a'],
tuple(['mac']): ['b'],
},
'bar_test': {
tuple(['win']): ['c'],
},
},
}
self.expectation_file = os.path.join(uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY,
'pixel_expectations.txt')
uu.CreateFile(self, self.expectation_file)
expectation_file_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
self._expectation_file_patcher = mock.patch.object(
uu.UnitTestExpectationProcessor, 'GetExpectationFileForSuite')
self._expectation_file_mock = self._expectation_file_patcher.start()
self._expectation_file_mock.return_value = self.expectation_file
self.addCleanup(self._expectation_file_patcher.stop)
def testGroupByTags(self) -> None:
"""Tests that threshold-based expectations work when grouping by tags."""
result_counts = {
tuple(['win']): {
# We expect this to be ignored since it has a 1% flake rate.
'foo_test': 100,
# We expect this to be RetryOnFailure since it has a 25% flake rate.
'bar_test': 4,
},
tuple(['mac']): {
# We expect this to be Failure since it has a 50% flake rate.
'foo_test': 2
}
}
self._expectations.IterateThroughResultsWithThresholds(
self.result_map, True, result_counts, 0.02, 0.5, True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ win ] bar_test [ RetryOnFailure ]
[ mac ] some_test [ Failure ]
[ mac ] foo_test [ Failure ]
[ android ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testNoGroupByTags(self) -> None:
"""Tests that threshold-based expectations work when not grouping by tags"""
result_counts = {
tuple(['win']): {
# We expect this to be ignored since it has a 1% flake rate.
'foo_test': 100,
# We expect this to be RetryOnFailure since it has a 25% flake rate.
'bar_test': 4,
},
tuple(['mac']): {
# We expect this to be Failure since it has a 50% flake rate.
'foo_test': 2
}
}
self._expectations.IterateThroughResultsWithThresholds(
self.result_map, False, result_counts, 0.02, 0.5, True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
[ mac ] foo_test [ Failure ]
[ win ] bar_test [ RetryOnFailure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testNoIncludeAllTags(self) -> None:
"""Tests that threshold-based expectations work when filtering tags."""
self.result_map = {
'pixel_integration_test': {
'foo_test': {
tuple(['win', 'win10']): ['a'],
tuple(['mac']): ['b'],
},
'bar_test': {
tuple(['win', 'win10']): ['c'],
},
},
}
result_counts = {
tuple(['win', 'win10']): {
# We expect this to be ignored since it has a 1% flake rate.
'foo_test': 100,
# We expect this to be RetryOnFailure since it has a 25% flake rate.
'bar_test': 4,
},
tuple(['mac']): {
# We expect this to be Failure since it has a 50% flake rate.
'foo_test': 2
}
}
self._expectations.IterateThroughResultsWithThresholds(
self.result_map, False, result_counts, 0.02, 0.5, False)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ android ] some_test [ Failure ]
[ mac ] foo_test [ Failure ]
[ win10 ] bar_test [ RetryOnFailure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
@unittest.skipIf(sys.version_info[0] != 3, 'Python 3-only')
class CreateExpectationsForAllResultsUnittest(fake_filesystem_unittest.TestCase
):
def setUp(self) -> None:
self.setUpPyfakefs()
self._expectations = uu.UnitTestExpectationProcessor()
self.result_map = {
'pixel_integration_test': {
'foo_test': {
tuple(['win']): [
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/1111',
datetime.date.today() - datetime.timedelta(days=2),
False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/2222',
datetime.date.today() - datetime.timedelta(days=3),
False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date.today(), False, ['Pass']),
],
tuple(['mac']): [
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/1111',
datetime.date.today() - datetime.timedelta(days=1),
False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/3333',
datetime.date.today() - datetime.timedelta(days=3),
False, ['Pass']),
],
},
'bar_test': {
tuple(['win']): [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/4444',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/5555',
datetime.date.today() - datetime.timedelta(days=1),
False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/6666',
datetime.date.today() - datetime.timedelta(days=2),
False, ['Pass']),
],
},
'baz_test': {
# This test config causes build fail on less than 2 consecutive
# days, and thus should not exist in the output.
tuple(['win']): [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/7777',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/8888',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/9999',
datetime.date.today(), False, ['Pass']),
],
tuple(['mac']): [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/7777',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/8888',
datetime.date.today(), False, ['Pass']),
],
},
'wpt_test': {
# Test for same test in all builders over threshold.
tuple(['win']): [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1234',
datetime.date.today(), False, ['Pass']),
],
tuple(['mac']): [
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/2345',
datetime.date.today() - datetime.timedelta(days=1),
False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3456',
datetime.date.today(), False, ['Pass']),
],
},
},
}
self.build_fail_total_number_threshold = 3
self.build_fail_consecutive_day_threshold = 2
self.build_fail_recent_day_threshold = 1
self.expectation_file = os.path.join(uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY,
'pixel_expectations.txt')
uu.CreateFile(self, self.expectation_file)
expectation_file_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure Pass ]
[ mac ] some_test [ Failure Pass ]
[ android ] some_test [ Failure Pass ]
"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
self._expectation_file_patcher = mock.patch.object(
uu.UnitTestExpectationProcessor, 'GetExpectationFileForSuite')
self._expectation_file_mock = self._expectation_file_patcher.start()
self._expectation_file_mock.return_value = self.expectation_file
self.addCleanup(self._expectation_file_patcher.stop)
def testGroupByTags(self) -> None:
"""Tests that threshold-based expectations work when grouping by tags."""
self._expectations.CreateExpectationsForAllResults(
self.result_map, True, True, self.build_fail_total_number_threshold,
self.build_fail_consecutive_day_threshold,
self.build_fail_recent_day_threshold)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure Pass ]
[ win ] foo_test [ Failure Pass ]
[ win ] bar_test [ Failure Pass ]
[ win ] wpt_test [ Failure Pass ]
[ mac ] some_test [ Failure Pass ]
[ mac ] foo_test [ Failure Pass ]
[ mac ] wpt_test [ Failure Pass ]
[ android ] some_test [ Failure Pass ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testNoGroupByTags(self) -> None:
"""Tests that threshold-based expectations work when not grouping by tags"""
self._expectations.CreateExpectationsForAllResults(
self.result_map, False, True, self.build_fail_total_number_threshold,
self.build_fail_consecutive_day_threshold,
self.build_fail_recent_day_threshold)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure Pass ]
[ mac ] some_test [ Failure Pass ]
[ android ] some_test [ Failure Pass ]
[ win ] foo_test [ Failure Pass ]
[ mac ] foo_test [ Failure Pass ]
[ win ] bar_test [ Failure Pass ]
[ win ] wpt_test [ Failure Pass ]
[ mac ] wpt_test [ Failure Pass ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testNoIncludeAllTags(self) -> None:
"""Tests that threshold-based expectations work when filtering tags."""
self.result_map = {
'pixel_integration_test': {
'foo_test': {
tuple(['win', 'win10']): [
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/1111',
datetime.date.today() - datetime.timedelta(days=2),
False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/2222',
datetime.date.today() - datetime.timedelta(days=3),
False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date.today(), False, ['Pass']),
],
tuple(['mac']): [
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/1111',
datetime.date.today() - datetime.timedelta(days=1),
False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/3333',
datetime.date.today() - datetime.timedelta(days=3),
False, ['Pass']),
],
},
'bar_test': {
tuple(['win', 'win10']): [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/4444',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/5555',
datetime.date.today() - datetime.timedelta(days=1),
False, ['Pass']),
ct.ResultTupleType(
ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/6666',
datetime.date.today() - datetime.timedelta(days=2),
False, ['Pass']),
],
},
'baz_test': {
# This test config causes build fail on less than 2 consecutive
# days, and thus should not exist in the output.
tuple(['win']): [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/7777',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/8888',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/9999',
datetime.date.today(), False, ['Pass']),
],
tuple(['mac']): [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/7777',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/8888',
datetime.date.today(), False, ['Pass']),
],
},
},
}
self._expectations.CreateExpectationsForAllResults(
self.result_map, False, False, self.build_fail_total_number_threshold,
self.build_fail_consecutive_day_threshold,
self.build_fail_recent_day_threshold)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure Pass ]
[ mac ] some_test [ Failure Pass ]
[ android ] some_test [ Failure Pass ]
[ win10 ] foo_test [ Failure Pass ]
[ mac ] foo_test [ Failure Pass ]
[ win10 ] bar_test [ Failure Pass ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
@unittest.skipIf(sys.version_info[0] != 3, 'Python 3-only')
class FindFailuresInSameConditionUnittest(unittest.TestCase):
def setUp(self) -> None:
self._expectations = uu.UnitTestExpectationProcessor()
self.result_map = {
'pixel_integration_test': {
'foo_test': {
tuple(['win']): ['a'],
tuple(['mac']): ['a', 'b'],
},
'bar_test': {
tuple(['win']): ['a', 'b', 'c'],
tuple(['mac']): ['a', 'b', 'c', 'd'],
},
},
'webgl_conformance_integration_test': {
'foo_test': {
tuple(['win']): ['a', 'b', 'c', 'd', 'e'],
tuple(['mac']): ['a', 'b', 'c', 'd', 'e', 'f'],
},
'bar_test': {
tuple(['win']): ['a', 'b', 'c', 'd', 'e', 'f', 'g'],
tuple(['mac']): ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
},
},
}
def testFindFailuresInSameTest(self) -> None:
other_failures = self._expectations.FindFailuresInSameTest(
self.result_map, 'pixel_integration_test', 'foo_test', tuple(['win']))
self.assertEqual(other_failures, [(tuple(['mac']), 2)])
def testFindFailuresInSameConfig(self) -> None:
typ_tag_ordered_result_map = self._expectations._ReorderMapByTypTags(
self.result_map)
other_failures = self._expectations.FindFailuresInSameConfig(
typ_tag_ordered_result_map, 'pixel_integration_test', 'foo_test',
tuple(['win']))
expected_other_failures = [
('pixel_integration_test.bar_test', 3),
('webgl_conformance_integration_test.foo_test', 5),
('webgl_conformance_integration_test.bar_test', 7),
]
self.assertEqual(len(other_failures), len(expected_other_failures))
self.assertEqual(set(other_failures), set(expected_other_failures))
@unittest.skipIf(sys.version_info[0] != 3, 'Python 3-only')
class ModifyFileForResultUnittest(fake_filesystem_unittest.TestCase):
def setUp(self) -> None:
self.setUpPyfakefs()
self._expectations = uu.UnitTestExpectationProcessor()
self.expectation_file = os.path.join(uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY,
'expectation.txt')
uu.CreateFile(self, self.expectation_file)
self._expectation_file_patcher = mock.patch.object(
uu.UnitTestExpectationProcessor, 'GetExpectationFileForSuite')
self._expectation_file_mock = self._expectation_file_patcher.start()
self.addCleanup(self._expectation_file_patcher.stop)
self._expectation_file_mock.return_value = self.expectation_file
def testNoGroupByTags(self) -> None:
"""Tests that not grouping by tags appends to the end."""
expectation_file_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
self._expectations.ModifyFileForResult('some_file', 'some_test',
('win', 'win10'), '', 'Failure',
False, True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ win win10 ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testGroupByTagsNoMatch(self) -> None:
"""Tests that grouping by tags but finding no match appends to the end."""
expectation_file_contents = uu.TAG_HEADER + """\
[ mac ] some_test [ Failure ]
"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
self._expectations.ModifyFileForResult('some_file', 'some_test',
('win', 'win10'), '', 'Failure',
True, True)
expected_contents = uu.TAG_HEADER + """\
[ mac ] some_test [ Failure ]
[ win win10 ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
def testGroupByTagsMatch(self) -> None:
"""Tests that grouping by tags and finding a match adds mid-file."""
expectation_file_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
self._expectations.ModifyFileForResult('some_file', 'foo_test',
('win', 'win10'), '', 'Failure',
True, True)
expected_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ win ] foo_test [ Failure ]
[ mac ] some_test [ Failure ]
"""
with open(self.expectation_file) as infile:
self.assertEqual(infile.read(), expected_contents)
@unittest.skipIf(sys.version_info[0] != 3, 'Python 3-only')
class FilterToMostSpecificTagTypeUnittest(fake_filesystem_unittest.TestCase):
def setUp(self) -> None:
self._expectations = uu.UnitTestExpectationProcessor()
self.setUpPyfakefs()
with tempfile.NamedTemporaryFile(delete=False) as tf:
self.expectation_file = tf.name
def testBasic(self):
"""Tests that only the most specific tags are kept."""
expectation_file_contents = """\
# tags: [ tag1_least_specific tag1_middle_specific tag1_most_specific ]
# tags: [ tag2_least_specific tag2_middle_specific tag2_most_specific ]"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
tags = ('tag1_least_specific', 'tag1_most_specific', 'tag2_middle_specific',
'tag2_least_specific')
filtered_tags = self._expectations.FilterToMostSpecificTypTags(
tags, self.expectation_file)
self.assertEqual(filtered_tags,
('tag1_most_specific', 'tag2_middle_specific'))
def testSingleTags(self) -> None:
"""Tests that functionality works as expected with single tags."""
expectation_file_contents = """\
# tags: [ tag1_most_specific ]
# tags: [ tag2_most_specific ]"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
tags = ('tag1_most_specific', 'tag2_most_specific')
filtered_tags = self._expectations.FilterToMostSpecificTypTags(
tags, self.expectation_file)
self.assertEqual(filtered_tags, tags)
def testUnusedTags(self) -> None:
"""Tests that functionality works as expected with extra/unused tags."""
expectation_file_contents = """\
# tags: [ tag1_least_specific tag1_middle_specific tag1_most_specific ]
# tags: [ tag2_least_specific tag2_middle_specific tag2_most_specific ]
# tags: [ some_unused_tag ]"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
tags = ('tag1_least_specific', 'tag1_most_specific', 'tag2_middle_specific',
'tag2_least_specific')
filtered_tags = self._expectations.FilterToMostSpecificTypTags(
tags, self.expectation_file)
self.assertEqual(filtered_tags,
('tag1_most_specific', 'tag2_middle_specific'))
def testMultiline(self) -> None:
"""Tests that functionality works when tags cover multiple lines."""
expectation_file_contents = """\
# tags: [ tag1_least_specific
# tag1_middle_specific
# tag1_most_specific ]
# tags: [ tag2_least_specific
# tag2_middle_specific tag2_most_specific ]"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
tags = ('tag1_least_specific', 'tag1_middle_specific', 'tag1_most_specific',
'tag2_middle_specific', 'tag2_least_specific')
filtered_tags = self._expectations.FilterToMostSpecificTypTags(
tags, self.expectation_file)
self.assertEqual(filtered_tags,
('tag1_most_specific', 'tag2_middle_specific'))
def testMissingTags(self) -> None:
"""Tests that a file not having all tags is an error."""
expectation_file_contents = """\
# tags: [ tag1_least_specific tag1_middle_specific ]
# tags: [ tag2_least_specific tag2_middle_specific tag2_most_specific ]"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
tags = ('tag1_least_specific', 'tag1_most_specific', 'tag2_middle_specific',
'tag2_least_specific')
with self.assertRaises(RuntimeError):
self._expectations.FilterToMostSpecificTypTags(tags,
self.expectation_file)
@unittest.skipIf(sys.version_info[0] != 3, 'Python 3-only')
class FindBestInsertionLineForExpectationUnittest(
fake_filesystem_unittest.TestCase):
def setUp(self) -> None:
self.setUpPyfakefs()
self._expectations = uu.UnitTestExpectationProcessor()
self.expectation_file = os.path.join(uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY,
'expectation.txt')
uu.CreateFile(self, self.expectation_file)
expectation_file_contents = uu.TAG_HEADER + """\
[ win ] some_test [ Failure ]
[ mac ] some_test [ Failure ]
[ win release ] bar_test [ Failure ]
[ win ] foo_test [ Failure ]
[ chromeos ] some_test [ Failure ]
"""
with open(self.expectation_file, 'w') as outfile:
outfile.write(expectation_file_contents)
def testNoMatchingTags(self) -> None:
"""Tests behavior when there are no expectations with matching tags."""
insertion_line, tags = (
self._expectations.FindBestInsertionLineForExpectation(
tuple(['android']), self.expectation_file))
self.assertEqual(insertion_line, -1)
self.assertEqual(tags, set())
def testMatchingTagsLastEntryChosen(self) -> None:
"""Tests that the last matching line is chosen."""
insertion_line, tags = (
self._expectations.FindBestInsertionLineForExpectation(
tuple(['win']), self.expectation_file))
# We expect "[ win ] foo_test [ Failure ]" to be chosen
expected_line = len(uu.TAG_HEADER.splitlines()) + 6
self.assertEqual(insertion_line, expected_line)
self.assertEqual(tags, set(['win']))
def testMatchingTagsClosestMatchChosen(self) -> None:
"""Tests that the closest tag match is chosen."""
insertion_line, tags = (
self._expectations.FindBestInsertionLineForExpectation(
('win', 'release'), self.expectation_file))
# We expect "[ win release ] bar_test [ Failure ]" to be chosen
expected_line = len(uu.TAG_HEADER.splitlines()) + 5
self.assertEqual(insertion_line, expected_line)
self.assertEqual(tags, set(['win', 'release']))
class AssertCheckoutIsUpToDateUnittest(unittest.TestCase):
def setUp(self) -> None:
self._expectations = uu.UnitTestExpectationProcessor()
self._origin_patcher = mock.patch(
'flake_suppressor_common.expectations.ExpectationProcessor.'
'GetOriginExpectationFileContents')
self._origin_mock = self._origin_patcher.start()
self.addCleanup(self._origin_patcher.stop)
self._local_patcher = mock.patch(
'flake_suppressor_common.expectations.' +
'ExpectationProcessor.GetLocalCheckoutExpectationFileContents')
self._local_mock = self._local_patcher.start()
self.addCleanup(self._local_patcher.stop)
def testContentsMatch(self) -> None:
"""Tests the happy path where the contents match."""
self._origin_mock.return_value = {
'foo.txt': 'foo_content',
'bar.txt': 'bar_content',
}
self._local_mock.return_value = {
'bar.txt': 'bar_content',
'foo.txt': 'foo_content',
}
self._expectations.AssertCheckoutIsUpToDate()
def testContentsDoNotMatch(self) -> None:
"""Tests that mismatched contents results in a failure."""
self._origin_mock.return_value = {
'foo.txt': 'foo_content',
'bar.txt': 'bar_content',
}
# Differing keys.
self._local_mock.return_value = {
'bar.txt': 'bar_content',
'foo2.txt': 'foo_content',
}
with self.assertRaises(RuntimeError):
self._expectations.AssertCheckoutIsUpToDate()
# Differing values.
self._local_mock.return_value = {
'bar.txt': 'bar_content',
'foo.txt': 'foo_content2',
}
with self.assertRaises(RuntimeError):
self._expectations.AssertCheckoutIsUpToDate()
class OverFailedBuildThresholdUnittest(unittest.TestCase):
def setUp(self) -> None:
self.build_fail_total_number_threshold = 3
def testOverThreshold(self) -> None:
"""Tests functionality when |result_tuple_list| passes
|build_fail_total_number_threshold|.
True is expected output on these inputs.
"""
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2021, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date(2023, 1, 1), False, ['Pass']),
]
self.assertTrue(
expectations.OverFailedBuildThreshold(
result_tuple_list, self.build_fail_total_number_threshold))
def testUnderThreshold(self) -> None:
"""Tests functionality when |result_tuple_list| cannot pass
|build_fail_total_number_threshold|.
False is expected output on these inputs.
"""
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 2), False, ['Pass']),
]
self.assertFalse(
expectations.OverFailedBuildThreshold(
result_tuple_list, self.build_fail_total_number_threshold))
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 2), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 3), False, ['Pass']),
]
self.assertFalse(
expectations.OverFailedBuildThreshold(
result_tuple_list, self.build_fail_total_number_threshold))
class OverFailedBuildByConsecutiveDayThresholdUnittest(unittest.TestCase):
def setUp(self) -> None:
self.build_fail_consecutive_day_threshold = 3
def testOverThreshold(self) -> None:
"""Tests functionality when |result_tuple_list| passes
|build_fail_consecutive_day_threshold|.
True is expected output on these inputs.
"""
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2022, 1, 2), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date(2022, 1, 3), False, ['Pass']),
]
self.assertTrue(
expectations.OverFailedBuildByConsecutiveDayThreshold(
result_tuple_list, self.build_fail_consecutive_day_threshold))
def testUnderThreshold(self) -> None:
"""Tests functionality when |result_tuple_list| cannot pass
|build_fail_consecutive_day_threshold|.
False is expected output on these inputs.
"""
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date(2022, 1, 1), False, ['Pass']),
]
self.assertFalse(
expectations.OverFailedBuildByConsecutiveDayThreshold(
result_tuple_list, self.build_fail_consecutive_day_threshold))
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 2), False, ['Pass']),
]
self.assertFalse(
expectations.OverFailedBuildByConsecutiveDayThreshold(
result_tuple_list, self.build_fail_consecutive_day_threshold))
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 2), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date(2022, 1, 4), False, ['Pass']),
]
self.assertFalse(
expectations.OverFailedBuildByConsecutiveDayThreshold(
result_tuple_list, self.build_fail_consecutive_day_threshold))
class FailedBuildWithinRecentDayThresholdUnittest(unittest.TestCase):
def setUp(self) -> None:
self.build_fail_recent_day_threshold = 3
def testWithinThreshold(self) -> None:
"""Tests functionality when |result_tuple_list| has build fail within
|build_fail_recent_day_threshold|.
True is expected output on these inputs.
"""
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date.today(), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date.today(), False, ['Pass']),
]
self.assertTrue(
expectations.FailedBuildWithinRecentDayThreshold(
result_tuple_list, self.build_fail_recent_day_threshold))
def testBeyondThreshold(self) -> None:
"""Tests functionality when |result_tuple_list| has no build fail within
|build_fail_recent_day_threshold|.
False is expected output on these inputs.
"""
result_tuple_list = [
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/1111',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/2222',
datetime.date(2022, 1, 1), False, ['Pass']),
ct.ResultTupleType(ct.ResultStatus.FAIL,
'http://ci.chromium.org/b/3333',
datetime.date(2022, 1, 1), False, ['Pass']),
]
self.assertFalse(
expectations.FailedBuildWithinRecentDayThreshold(
result_tuple_list, self.build_fail_recent_day_threshold))
if __name__ == '__main__':
unittest.main(verbosity=2)
Codebase Browser
chromium