results_merger_test.py | Explore in Territory

#!/usr/bin/env vpython3
# Copyright 2016 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import copy
import unittest

import six

import results_merger

GOOD_JSON_TEST_RESULT_0 = {
    'tests': {
        'car': {
            'honda': {
                'expected': 'PASS',
                'actual': 'PASS'
            },
            'toyota': {
                'expected': 'FAIL',
                'actual': 'FAIL'
            }
        },
        'computer': {
            'dell': {
                'expected': 'PASS',
                'actual': 'PASS'
            }
        },
    },
    'interrupted': False,
    'version': 3,
    'seconds_since_epoch': 1406662289.76,
    'num_failures_by_type': {
        'FAIL': 0,
        'PASS': 2
    },
    'layout_tests_dir': 'abc'
}

GOOD_JSON_TEST_RESULT_1 = {
    'tests': {
        'car': {
            'tesla': {
                'expected': 'PASS',
                'actual': 'PASS'
            },
        },
        'burger': {
            'mcdonald': {
                'expected': 'PASS',
                'actual': 'PASS'
            }
        },
    },
    'interrupted': False,
    'version': 3,
    'seconds_since_epoch': 1406662283.11,
    'num_failures_by_type': {
        'FAIL': 0,
        'PASS': 2
    },
    'layout_tests_dir': '123'
}

GOOD_JSON_TEST_RESULT_2 = {
    'tests': {
        'car': {
            'mercedes': {
                'expected': 'PASS',
                'actual': 'FAIL'
            },
        },
        'burger': {
            'in n out': {
                'expected': 'PASS',
                'actual': 'PASS'
            }
        },
    },
    'interrupted': True,
    'version': 3,
    'seconds_since_epoch': 1406662200.01,
    'num_failures_by_type': {
        'FAIL': 1,
        'PASS': 1
    }
}

GOOD_JSON_TEST_RESULT_MERGED = {
    'tests': {
        'car': {
            'tesla': {
                'expected': 'PASS',
                'actual': 'PASS'
            },
            'mercedes': {
                'expected': 'PASS',
                'actual': 'FAIL'
            },
            'honda': {
                'expected': 'PASS',
                'actual': 'PASS'
            },
            'toyota': {
                'expected': 'FAIL',
                'actual': 'FAIL'
            }
        },
        'computer': {
            'dell': {
                'expected': 'PASS',
                'actual': 'PASS'
            }
        },
        'burger': {
            'mcdonald': {
                'expected': 'PASS',
                'actual': 'PASS'
            },
            'in n out': {
                'expected': 'PASS',
                'actual': 'PASS'
            }
        }
    },
    'interrupted': True,
    'version': 3,
    'seconds_since_epoch': 1406662200.01,
    'num_failures_by_type': {
        'FAIL': 1,
        'PASS': 5
    },
    'layout_tests_dir': '123'
}


def extend(initial, add):
  out = copy.deepcopy(initial)
  out.update(add)
  return out


def remove(initial, keys):
  out = copy.deepcopy(initial)
  for k in keys:
    del out[k]
  return out


# These unittests are run in PRESUBMIT, but not by recipe_simulation_test, hence
# to avoid false alert on missing coverage by recipe_simulation_test, we mark
# these code as no cover.
class MergingTest(unittest.TestCase):  # pragma: no cover
  maxDiff = None  # Show full diff if assertion fail

  def test_merge_tries(self):
    self.assertEqual({
        'a': 'A',
        'b': {
            'c': 'C'
        }
    }, results_merger.merge_tries({
        'a': 'A',
        'b': {}
    }, {'b': {
        'c': 'C'
    }}))

  def test_merge_tries_unmergable(self):
    with six.assertRaisesRegex(self, results_merger.MergeException, 'a:b'):
      results_merger.merge_tries({'a': {'b': 'A'}}, {'a': {'b': 'C'}})

  def test_merge_metadata(self):
    metadata1 = {'metadata': {'tags': ['foo', 'bar']}}
    metadata2 = {'metadata': {'tags': ['foo', 'bat']}}
    merged_results = results_merger.merge_test_results([
        extend(GOOD_JSON_TEST_RESULT_0, metadata1),
        extend(GOOD_JSON_TEST_RESULT_1, metadata2)
    ])
    self.assertEqual(merged_results['metadata']['tags'], ['foo', 'bat'])

  def test_merge_json_test_results_nop(self):
    good_json_results = (GOOD_JSON_TEST_RESULT_0, GOOD_JSON_TEST_RESULT_1,
                         GOOD_JSON_TEST_RESULT_2, GOOD_JSON_TEST_RESULT_MERGED)
    for j in good_json_results:
      # Clone so we can check the input dictionaries are not modified
      a = copy.deepcopy(j)
      self.assertEqual(results_merger.merge_test_results([a]), j)
      self.assertEqual(a, j)

  def test_merge_json_test_results_invalid_version(self):
    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          extend(GOOD_JSON_TEST_RESULT_0, {'version': 5}),
      ])

    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          GOOD_JSON_TEST_RESULT_0,
          extend(GOOD_JSON_TEST_RESULT_1, {'version': 5}),
      ])

  def test_merge_json_test_results_missing_version(self):
    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          remove(GOOD_JSON_TEST_RESULT_0, ['version']),
      ])

    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          GOOD_JSON_TEST_RESULT_0,
          remove(GOOD_JSON_TEST_RESULT_1, ['version']),
      ])

  def test_merge_json_test_results_invalid_extra(self):
    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          extend(GOOD_JSON_TEST_RESULT_0, {'extra': True}),
      ])

    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          GOOD_JSON_TEST_RESULT_0,
          extend(GOOD_JSON_TEST_RESULT_1, {'extra': True}),
      ])

  def test_merge_json_test_results_missing_required(self):
    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          remove(GOOD_JSON_TEST_RESULT_0, ['interrupted']),
      ])

    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          GOOD_JSON_TEST_RESULT_0,
          remove(GOOD_JSON_TEST_RESULT_1, ['interrupted']),
      ])

  def test_merge_json_test_results_multiple(self):
    self.assertEqual(
        results_merger.merge_test_results([
            GOOD_JSON_TEST_RESULT_0,
            GOOD_JSON_TEST_RESULT_1,
            GOOD_JSON_TEST_RESULT_2,
        ]), GOOD_JSON_TEST_RESULT_MERGED)

  def test_merge_json_test_results_optional_matches(self):
    self.assertEqual(
        results_merger.merge_test_results([
            extend(GOOD_JSON_TEST_RESULT_0, {'path_delimiter': '.'}),
            extend(GOOD_JSON_TEST_RESULT_1, {'path_delimiter': '.'}),
            extend(GOOD_JSON_TEST_RESULT_2, {'path_delimiter': '.'}),
        ]), extend(GOOD_JSON_TEST_RESULT_MERGED, {'path_delimiter': '.'}))

  def test_merge_json_test_results_optional_differs(self):
    with self.assertRaises(results_merger.MergeException):
      results_merger.merge_test_results([
          extend(GOOD_JSON_TEST_RESULT_0, {'path_delimiter': '.'}),
          extend(GOOD_JSON_TEST_RESULT_1, {'path_delimiter': '.'}),
          extend(GOOD_JSON_TEST_RESULT_2, {'path_delimiter': '/'}),
      ])

  def test_merge_json_test_results_optional_count(self):
    self.assertEqual(
        results_merger.merge_test_results([
            extend(GOOD_JSON_TEST_RESULT_0, {'fixable': 1}),
            extend(GOOD_JSON_TEST_RESULT_1, {'fixable': 2}),
            extend(GOOD_JSON_TEST_RESULT_2, {'fixable': 3}),
        ]), extend(GOOD_JSON_TEST_RESULT_MERGED, {'fixable': 6}))

  def test_merge_nothing(self):
    self.assertEqual(results_merger.merge_test_results([]), {})


# TODO(tansell): Make this test fail properly, currently fails with an
# AttributeError.
#  def test_merge_test_name_conflict(self):
#    self.maxDiff = None  # Show full diff if assertion fail
#    with self.assertRaises(results_merger.MergeException):
#      results_merger.merge_test_results(
#        [GOOD_JSON_TEST_RESULT_0, GOOD_JSON_TEST_RESULT_0])

if __name__ == '__main__':
  unittest.main()  # pragma: no cover
chromium/testing/merge_scripts/results_merger_test.py
