# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Most of this file was ported over from Blink's
# tools/blinkpy/web_tests/layout_package/json_results_generator.py
# tools/blinkpy/common/net/file_uploader.py
#
import json
import logging
import mimetypes
import os
import time
try:
from urllib.request import urlopen, Request
from urllib.error import HTTPError, URLError
from urllib.parse import quote
except ImportError:
from urllib import quote
from urllib2 import urlopen, HTTPError, URLError, Request
_log = logging.getLogger(__name__)
_JSON_PREFIX = 'ADD_RESULTS('
_JSON_SUFFIX = ');'
def HasJSONWrapper(string):
return string.startswith(_JSON_PREFIX) and string.endswith(_JSON_SUFFIX)
def StripJSONWrapper(json_content):
# FIXME: Kill this code once the server returns json instead of jsonp.
if HasJSONWrapper(json_content):
return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)]
return json_content
def WriteJSON(json_object, file_path, callback=None):
# Specify separators in order to get compact encoding.
json_string = json.dumps(json_object, separators=(',', ':'))
if callback:
json_string = callback + '(' + json_string + ');'
with open(file_path, 'w') as fp:
fp.write(json_string)
def ConvertTrieToFlatPaths(trie, prefix=None):
"""Flattens the trie of paths, prepending a prefix to each."""
result = {}
for name, data in trie.items():
if prefix:
name = prefix + '/' + name
if len(data) != 0 and not 'results' in data:
result.update(ConvertTrieToFlatPaths(data, name))
else:
result[name] = data
return result
def AddPathToTrie(path, value, trie):
"""Inserts a single path and value into a directory trie structure."""
if not '/' in path:
trie[path] = value
return
directory, _, rest = path.partition('/')
if not directory in trie:
trie[directory] = {}
AddPathToTrie(rest, value, trie[directory])
def TestTimingsTrie(individual_test_timings):
"""Breaks a test name into dicts by directory
foo/bar/baz.html: 1ms
foo/bar/baz1.html: 3ms
becomes
foo: {
bar: {
baz.html: 1,
baz1.html: 3
}
}
"""
trie = {}
for test_result in individual_test_timings:
test = test_result.test_name
AddPathToTrie(test, int(1000 * test_result.test_run_time), trie)
return trie
class TestResult:
"""A simple class that represents a single test result."""
# Test modifier constants.
(NONE, FAILS, FLAKY, DISABLED) = list(range(4))
def __init__(self, test, failed=False, elapsed_time=0):
self.test_name = test
self.failed = failed
self.test_run_time = elapsed_time
test_name = test
try:
test_name = test.split('.')[1]
except IndexError:
_log.warning('Invalid test name: %s.', test)
if test_name.startswith('FAILS_'):
self.modifier = self.FAILS
elif test_name.startswith('FLAKY_'):
self.modifier = self.FLAKY
elif test_name.startswith('DISABLED_'):
self.modifier = self.DISABLED
else:
self.modifier = self.NONE
def Fixable(self):
return self.failed or self.modifier == self.DISABLED
class JSONResultsGeneratorBase:
"""A JSON results generator for generic tests."""
MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750
# Min time (seconds) that will be added to the JSON.
MIN_TIME = 1
# Note that in non-chromium tests those chars are used to indicate
# test modifiers (FAILS, FLAKY, etc) but not actual test results.
PASS_RESULT = 'P'
SKIP_RESULT = 'X'
FAIL_RESULT = 'F'
FLAKY_RESULT = 'L'
NO_DATA_RESULT = 'N'
MODIFIER_TO_CHAR = {TestResult.NONE: PASS_RESULT,
TestResult.DISABLED: SKIP_RESULT,
TestResult.FAILS: FAIL_RESULT,
TestResult.FLAKY: FLAKY_RESULT}
VERSION = 4
VERSION_KEY = 'version'
RESULTS = 'results'
TIMES = 'times'
BUILD_NUMBERS = 'buildNumbers'
TIME = 'secondsSinceEpoch'
TESTS = 'tests'
FIXABLE_COUNT = 'fixableCount'
FIXABLE = 'fixableCounts'
ALL_FIXABLE_COUNT = 'allFixableCount'
RESULTS_FILENAME = 'results.json'
TIMES_MS_FILENAME = 'times_ms.json'
INCREMENTAL_RESULTS_FILENAME = 'incremental_results.json'
# line too long pylint: disable=line-too-long
URL_FOR_TEST_LIST_JSON = (
'https://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s&'
'master=%s')
# pylint: enable=line-too-long
def __init__(self, builder_name, build_name, build_number,
results_file_base_path, builder_base_url,
test_results_map, svn_repositories=None,
test_results_server=None,
test_type='',
master_name=''):
"""Modifies the results.json file. Grabs it off the archive directory
if it is not found locally.
Args
builder_name: the builder name (e.g. Webkit).
build_name: the build name (e.g. webkit-rel).
build_number: the build number.
results_file_base_path: Absolute path to the directory containing the
results json file.
builder_base_url: the URL where we have the archived test results.
If this is None no archived results will be retrieved.
test_results_map: A dictionary that maps test_name to TestResult.
svn_repositories: A (json_field_name, svn_path) pair for SVN
repositories that tests rely on. The SVN revision will be
included in the JSON with the given json_field_name.
test_results_server: server that hosts test results json.
test_type: test type string (e.g. 'layout-tests').
master_name: the name of the buildbot master.
"""
self._builder_name = builder_name
self._build_name = build_name
self._build_number = build_number
self._builder_base_url = builder_base_url
self._results_directory = results_file_base_path
self._test_results_map = test_results_map
self._test_results = list(test_results_map.values())
self._svn_repositories = svn_repositories
if not self._svn_repositories:
self._svn_repositories = {}
self._test_results_server = test_results_server
self._test_type = test_type
self._master_name = master_name
self._archived_results = None
def GenerateJSONOutput(self):
json_object = self.GetJSON()
if json_object:
file_path = (
os.path.join(
self._results_directory,
self.INCREMENTAL_RESULTS_FILENAME))
WriteJSON(json_object, file_path)
def GenerateTimesMSFile(self):
times = TestTimingsTrie(list(self._test_results_map.values()))
file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME)
WriteJSON(times, file_path)
def GetJSON(self):
"""Gets the results for the results.json file."""
results_json = {}
if not results_json:
results_json, error = self._GetArchivedJSONResults()
if error:
# If there was an error don't write a results.json
# file at all as it would lose all the information on the
# bot.
_log.error(
'Archive directory is inaccessible. Not '
'modifying or clobbering the results.json '
'file: %s', error)
return None
builder_name = self._builder_name
if results_json and builder_name not in results_json:
_log.debug('Builder name (%s) is not in the results.json file.',
builder_name)
self._ConvertJSONToCurrentVersion(results_json)
if builder_name not in results_json:
results_json[builder_name] = (
self._CreateResultsForBuilderJSON())
results_for_builder = results_json[builder_name]
if builder_name:
self._InsertGenericMetaData(results_for_builder)
self._InsertFailureSummaries(results_for_builder)
# Update the all failing tests with result type and time.
tests = results_for_builder[self.TESTS]
all_failing_tests = self._GetFailedTestNames()
all_failing_tests.update(ConvertTrieToFlatPaths(tests))
for test in all_failing_tests:
self._InsertTestTimeAndResult(test, tests)
return results_json
def SetArchivedResults(self, archived_results):
self._archived_results = archived_results
def UploadJSONFiles(self, json_files):
"""Uploads the given json_files to the test_results_server (if the
test_results_server is given)."""
if not self._test_results_server:
return
if not self._master_name:
_log.error(
'--test-results-server was set, but --master-name was not. Not '
'uploading JSON files.')
return
_log.info('Uploading JSON files for builder: %s', self._builder_name)
attrs = [('builder', self._builder_name),
('testtype', self._test_type),
('master', self._master_name)]
files = [(json_file, os.path.join(self._results_directory, json_file))
for json_file in json_files]
url = 'https://%s/testfile/upload' % self._test_results_server
# Set uploading timeout in case appengine server is having problems.
# 120 seconds are more than enough to upload test results.
uploader = _FileUploader(url, 120)
try:
response = uploader.UploadAsMultipartFormData(files, attrs)
if response:
if response.code == 200:
_log.info('JSON uploaded.')
else:
_log.debug(
"JSON upload failed, %d: '%s'", response.code, response.read())
else:
_log.error('JSON upload failed; no response returned')
except Exception as err: # pylint: disable=broad-except
_log.error('Upload failed: %s', err)
return
def _GetTestTiming(self, test_name):
"""Returns test timing data (elapsed time) in second
for the given test_name."""
if test_name in self._test_results_map:
# Floor for now to get time in seconds.
return int(self._test_results_map[test_name].test_run_time)
return 0
def _GetFailedTestNames(self):
"""Returns a set of failed test names."""
return set(r.test_name for r in self._test_results if r.failed)
def _GetModifierChar(self, test_name):
"""Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT,
PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test modifier
for the given test_name.
"""
if test_name not in self._test_results_map:
return self.__class__.NO_DATA_RESULT
test_result = self._test_results_map[test_name]
if test_result.modifier in list(self.MODIFIER_TO_CHAR.keys()):
return self.MODIFIER_TO_CHAR[test_result.modifier]
return self.__class__.PASS_RESULT
def _get_result_char(self, test_name):
"""Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT,
PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result
for the given test_name.
"""
if test_name not in self._test_results_map:
return self.__class__.NO_DATA_RESULT
test_result = self._test_results_map[test_name]
if test_result.modifier == TestResult.DISABLED:
return self.__class__.SKIP_RESULT
if test_result.failed:
return self.__class__.FAIL_RESULT
return self.__class__.PASS_RESULT
def _GetSVNRevision(self, in_directory):
"""Returns the svn revision for the given directory.
Args:
in_directory: The directory where svn is to be run.
"""
# This is overridden in flakiness_dashboard_results_uploader.py.
raise NotImplementedError()
def _GetArchivedJSONResults(self):
"""Download JSON file that only contains test
name list from test-results server. This is for generating incremental
JSON so the file generated has info for tests that failed before but
pass or are skipped from current run.
Returns (archived_results, error) tuple where error is None if results
were successfully read.
"""
results_json = {}
old_results = None
error = None
if not self._test_results_server:
return {}, None
results_file_url = (self.URL_FOR_TEST_LIST_JSON %
(quote(self._test_results_server),
quote(self._builder_name), self.RESULTS_FILENAME,
quote(self._test_type), quote(self._master_name)))
try:
# FIXME: We should talk to the network via a Host object.
results_file = urlopen(results_file_url)
old_results = results_file.read()
except HTTPError as http_error:
# A non-4xx status code means the bot is hosed for some reason
# and we can't grab the results.json file off of it.
if http_error.code < 400 and http_error.code >= 500:
error = http_error
except URLError as url_error:
error = url_error
if old_results:
# Strip the prefix and suffix so we can get the actual JSON object.
old_results = StripJSONWrapper(old_results)
try:
results_json = json.loads(old_results)
except Exception: # pylint: disable=broad-except
_log.debug('results.json was not valid JSON. Clobbering.')
# The JSON file is not valid JSON. Just clobber the results.
results_json = {}
else:
_log.debug('Old JSON results do not exist. Starting fresh.')
results_json = {}
return results_json, error
def _InsertFailureSummaries(self, results_for_builder):
"""Inserts aggregate pass/failure statistics into the JSON.
This method reads self._test_results and generates
FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries.
Args:
results_for_builder: Dictionary containing the test results for a
single builder.
"""
# Insert the number of tests that failed or skipped.
fixable_count = len([r for r in self._test_results if r.Fixable()])
self._InsertItemIntoRawList(results_for_builder,
fixable_count, self.FIXABLE_COUNT)
# Create a test modifiers (FAILS, FLAKY etc) summary dictionary.
entry = {}
for test_name in self._test_results_map.keys():
result_char = self._GetModifierChar(test_name)
entry[result_char] = entry.get(result_char, 0) + 1
# Insert the pass/skip/failure summary dictionary.
self._InsertItemIntoRawList(results_for_builder, entry,
self.FIXABLE)
# Insert the number of all the tests that are supposed to pass.
all_test_count = len(self._test_results)
self._InsertItemIntoRawList(results_for_builder,
all_test_count, self.ALL_FIXABLE_COUNT)
def _InsertItemIntoRawList(self, results_for_builder, item, key):
"""Inserts the item into the list with the given key in the results for
this builder. Creates the list if no such list exists.
Args:
results_for_builder: Dictionary containing the test results for a
single builder.
item: Number or string to insert into the list.
key: Key in results_for_builder for the list to insert into.
"""
if key in results_for_builder:
raw_list = results_for_builder[key]
else:
raw_list = []
raw_list.insert(0, item)
raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG]
results_for_builder[key] = raw_list
def _InsertItemRunLengthEncoded(self, item, encoded_results):
"""Inserts the item into the run-length encoded results.
Args:
item: String or number to insert.
encoded_results: run-length encoded results. An array of arrays, e.g.
[[3,'A'],[1,'Q']] encodes AAAQ.
"""
if len(encoded_results) != 0 and item == encoded_results[0][1]:
num_results = encoded_results[0][0]
if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
encoded_results[0][0] = num_results + 1
else:
# Use a list instead of a class for the run-length encoding since
# we want the serialized form to be concise.
encoded_results.insert(0, [1, item])
def _InsertGenericMetaData(self, results_for_builder):
""" Inserts generic metadata (such as version number, current time etc)
into the JSON.
Args:
results_for_builder: Dictionary containing the test results for
a single builder.
"""
self._InsertItemIntoRawList(results_for_builder,
self._build_number, self.BUILD_NUMBERS)
# Include SVN revisions for the given repositories.
for (name, path) in self._svn_repositories:
# Note: for JSON file's backward-compatibility we use 'chrome' rather
# than 'chromium' here.
lowercase_name = name.lower()
if lowercase_name == 'chromium':
lowercase_name = 'chrome'
self._InsertItemIntoRawList(results_for_builder,
self._GetSVNRevision(path),
lowercase_name + 'Revision')
self._InsertItemIntoRawList(results_for_builder,
int(time.time()),
self.TIME)
def _InsertTestTimeAndResult(self, test_name, tests):
""" Insert a test item with its results to the given tests dictionary.
Args:
tests: Dictionary containing test result entries.
"""
result = self._get_result_char(test_name)
test_time = self._GetTestTiming(test_name)
this_test = tests
for segment in test_name.split('/'):
if segment not in this_test:
this_test[segment] = {}
this_test = this_test[segment]
if len(this_test) == 0:
self._PopulateResultsAndTimesJSON(this_test)
if self.RESULTS in this_test:
self._InsertItemRunLengthEncoded(result, this_test[self.RESULTS])
else:
this_test[self.RESULTS] = [[1, result]]
if self.TIMES in this_test:
self._InsertItemRunLengthEncoded(test_time, this_test[self.TIMES])
else:
this_test[self.TIMES] = [[1, test_time]]
def _ConvertJSONToCurrentVersion(self, results_json):
"""If the JSON does not match the current version, converts it to the
current version and adds in the new version number.
"""
if self.VERSION_KEY in results_json:
archive_version = results_json[self.VERSION_KEY]
if archive_version == self.VERSION:
return
else:
archive_version = 3
# version 3->4
if archive_version == 3:
for results in list(results_json.values()):
self._ConvertTestsToTrie(results)
results_json[self.VERSION_KEY] = self.VERSION
def _ConvertTestsToTrie(self, results):
if not self.TESTS in results:
return
test_results = results[self.TESTS]
test_results_trie = {}
for test in test_results.keys():
single_test_result = test_results[test]
AddPathToTrie(test, single_test_result, test_results_trie)
results[self.TESTS] = test_results_trie
def _PopulateResultsAndTimesJSON(self, results_and_times):
results_and_times[self.RESULTS] = []
results_and_times[self.TIMES] = []
return results_and_times
def _CreateResultsForBuilderJSON(self):
results_for_builder = {}
results_for_builder[self.TESTS] = {}
return results_for_builder
def _RemoveItemsOverMaxNumberOfBuilds(self, encoded_list):
"""Removes items from the run-length encoded list after the final
item that exceeds the max number of builds to track.
Args:
encoded_results: run-length encoded results. An array of arrays, e.g.
[[3,'A'],[1,'Q']] encodes AAAQ.
"""
num_builds = 0
index = 0
for result in encoded_list:
num_builds = num_builds + result[0]
index = index + 1
if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
return encoded_list[:index]
return encoded_list
def _NormalizeResultsJSON(self, test, test_name, tests):
""" Prune tests where all runs pass or tests that no longer exist and
truncate all results to maxNumberOfBuilds.
Args:
test: ResultsAndTimes object for this test.
test_name: Name of the test.
tests: The JSON object with all the test results for this builder.
"""
test[self.RESULTS] = self._RemoveItemsOverMaxNumberOfBuilds(
test[self.RESULTS])
test[self.TIMES] = self._RemoveItemsOverMaxNumberOfBuilds(
test[self.TIMES])
is_all_pass = self._IsResultsAllOfType(test[self.RESULTS],
self.PASS_RESULT)
is_all_no_data = self._IsResultsAllOfType(test[self.RESULTS],
self.NO_DATA_RESULT)
max_time = max([test_time[1] for test_time in test[self.TIMES]])
# Remove all passes/no-data from the results to reduce noise and
# filesize. If a test passes every run, but takes > MIN_TIME to run,
# don't throw away the data.
if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME):
del tests[test_name]
# method could be a function pylint: disable=R0201
def _IsResultsAllOfType(self, results, result_type):
"""Returns whether all the results are of the given type
(e.g. all passes)."""
return len(results) == 1 and results[0][1] == result_type
class _FileUploader:
def __init__(self, url, timeout_seconds):
self._url = url
self._timeout_seconds = timeout_seconds
def UploadAsMultipartFormData(self, files, attrs):
file_objs = []
for filename, path in files:
with open(path, 'rb') as fp:
file_objs.append(('file', filename, fp.read()))
# FIXME: We should use the same variable names for the formal and actual
# parameters.
content_type, data = _EncodeMultipartFormData(attrs, file_objs)
return self._UploadData(content_type, data)
def _UploadData(self, content_type, data):
start = time.time()
end = start + self._timeout_seconds
while time.time() < end:
try:
request = Request(self._url, data, {'Content-Type': content_type})
return urlopen(request)
except HTTPError as e:
_log.warning(
'Received HTTP status %s loading "%s". '
'Retrying in 10 seconds...', e.code, e.filename)
time.sleep(10)
def _GetMIMEType(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
# FIXME: Rather than taking tuples, this function should take more
# structured data.
def _EncodeMultipartFormData(fields, files):
"""Encode form fields for multipart/form-data.
Args:
fields: A sequence of (name, value) elements for regular form fields.
files: A sequence of (name, filename, value) elements for data to be
uploaded as files.
Returns:
(content_type, body) ready for httplib.HTTP instance.
Source:
http://code.google.com/p/rietveld/source/browse/trunk/upload.py
"""
BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
CRLF = '\r\n'
lines = []
for key, value in fields:
lines.append('--' + BOUNDARY)
lines.append('Content-Disposition: form-data; name="%s"' % key)
lines.append('')
if isinstance(value, str):
value = value.encode('utf-8')
lines.append(value)
for key, filename, value in files:
lines.append('--' + BOUNDARY)
lines.append('Content-Disposition: form-data; name="%s"; '
'filename="%s"' % (key, filename))
lines.append('Content-Type: %s' % _GetMIMEType(filename))
lines.append('')
if isinstance(value, str):
value = value.encode('utf-8')
lines.append(value)
lines.append('--' + BOUNDARY + '--')
lines.append('')
body = CRLF.join(lines)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
Codebase Browser
chromium