#!/usr/bin/env python
# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Tool for seeing the real world impact of a patch.
#
# Layout Tests can tell you whether something has changed, but this can help
# you determine whether a subtle/controversial change is beneficial or not.
#
# It dumps the rendering of a large number of sites, both with and without a
# patch being evaluated, then sorts them by greatest difference in rendering,
# such that a human reviewer can quickly review the most impacted sites,
# rather than having to manually try sites to see if anything changes.
#
# In future it might be possible to extend this to other kinds of differences,
# e.g. page load times.
from __future__ import print_function
import argparse
from argparse import RawTextHelpFormatter
from contextlib import closing
import datetime
import errno
from distutils.spawn import find_executable
from operator import itemgetter
import multiprocessing
import os
import re
from cStringIO import StringIO
import subprocess
import sys
import textwrap
import time
from urllib2 import urlopen
from urlparse import urlparse
import webbrowser
from zipfile import ZipFile
from nsfw_urls import nsfw_urls
action = None
allow_js = False
additional_content_shell_flags = ""
chromium_src_root = ""
chromium_out_dir = ""
image_diff = ""
content_shell = ""
output_dir = ""
num_sites = 100
urls = []
print_lock = multiprocessing.Lock()
def MakeDirsIfNotExist(dir):
try:
os.makedirs(dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
def SetupPathsAndOut():
global chromium_src_root, chromium_out_dir, output_dir
global image_diff, content_shell
chromium_src_root = os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
# Find out directory (might be out_linux for users of cr).
for out_suffix in ["_linux", ""]:
out_dir = os.path.join(chromium_src_root, "out" + out_suffix)
if os.path.exists(out_dir):
chromium_out_dir = out_dir
break
if not chromium_out_dir:
return False
this_script_name = "real_world_impact"
output_dir = os.path.join(chromium_out_dir,
"Release",
this_script_name)
MakeDirsIfNotExist(output_dir)
image_diff = os.path.join(chromium_out_dir, "Release", "image_diff")
if sys.platform == 'darwin':
content_shell = os.path.join(chromium_out_dir, "Release",
"Content Shell.app/Contents/MacOS/Content Shell")
elif sys.platform.startswith('linux'):
content_shell = os.path.join(chromium_out_dir, "Release",
"content_shell")
elif sys.platform.startswith('win'):
content_shell = os.path.join(chromium_out_dir, "Release",
"content_shell.exe")
return True
def CheckPrerequisites():
if not find_executable("wget"):
print("wget not found! Install wget and re-run this.")
return False
if not os.path.exists(image_diff):
print("image_diff not found (%s)!" % image_diff)
print("Build the image_diff target and re-run this.")
return False
if not os.path.exists(content_shell):
print("Content shell not found (%s)!" % content_shell)
print("Build Release/content_shell and re-run this.")
return False
return True
def PickSampleUrls():
global urls
data_dir = os.path.join(output_dir, "data")
MakeDirsIfNotExist(data_dir)
# Download Alexa top 1,000,000 sites
# TODO(johnme): Should probably update this when it gets too stale...
csv_path = os.path.join(data_dir, "top-1m.csv")
if not os.path.exists(csv_path):
print("Downloading list of top 1,000,000 sites from Alexa...")
csv_url = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
with closing(urlopen(csv_url)) as stream:
ZipFile(StringIO(stream.read())).extract("top-1m.csv", data_dir)
bad_urls_path = os.path.join(data_dir, "bad_urls.txt")
if os.path.exists(bad_urls_path):
with open(bad_urls_path) as f:
bad_urls = set(f.read().splitlines())
else:
bad_urls = set()
# See if we've already selected a sample of size num_sites (this way, if you
# call this script with arguments "before N" then "after N", where N is the
# same number, we'll use the same sample, as expected!).
urls_path = os.path.join(data_dir, "%06d_urls.txt" % num_sites)
if not os.path.exists(urls_path):
if action == 'compare':
print("Error: you must run 'before %d' and 'after %d' before "
"running 'compare %d'" % (num_sites, num_sites, num_sites))
return False
print("Picking %d sample urls..." % num_sites)
# TODO(johnme): For now this just gets the top num_sites entries. In future
# this should pick a weighted random sample. For example, it could fit a
# power-law distribution, which is a good model of website popularity
# (http://www.useit.com/alertbox/9704b.html).
urls = []
remaining_num_sites = num_sites
with open(csv_path) as f:
for entry in f:
if remaining_num_sites <= 0:
break
remaining_num_sites -= 1
hostname = entry.strip().split(',')[1]
if not '/' in hostname: # Skip Alexa 1,000,000 entries that have paths.
url = "http://%s/" % hostname
if not url in bad_urls:
urls.append(url)
# Don't write these to disk yet; we'll do that in SaveWorkingUrls below
# once we have tried to download them and seen which ones fail.
else:
with open(urls_path) as f:
urls = [u for u in f.read().splitlines() if not u in bad_urls]
return True
def SaveWorkingUrls():
# TODO(johnme): Update the list if a url that used to work goes offline.
urls_path = os.path.join(output_dir, "data", "%06d_urls.txt" % num_sites)
if not os.path.exists(urls_path):
with open(urls_path, 'w') as f:
f.writelines(u + '\n' for u in urls)
def PrintElapsedTime(elapsed, detail=""):
elapsed = round(elapsed * 10) / 10.0
m = elapsed / 60
s = elapsed % 60
print("Took %dm%.1fs" % (m, s), detail)
def DownloadStaticCopyTask(url):
url_parts = urlparse(url)
host_dir = os.path.join(output_dir, "data", url_parts.hostname)
# Use wget for now, as does a reasonable job of spidering page dependencies
# (e.g. CSS, JS, images).
success = True
try:
subprocess.check_call(["wget",
"--execute", "robots=off",
("--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS "
"X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) C"
"hrome/32.0.1700.14 Safari/537.36"),
"--page-requisites",
"--span-hosts",
"--adjust-extension",
"--convert-links",
"--directory-prefix=" + host_dir,
"--force-directories",
"--default-page=index.html",
"--no-check-certificate",
"--timeout=5", # 5s timeout
"--tries=2",
"--quiet",
url])
except KeyboardInterrupt:
success = False
except subprocess.CalledProcessError:
# Ignoring these for now, as some sites have issues with their subresources
# yet still produce a renderable index.html
pass #success = False
if success:
download_path = os.path.join(host_dir, url_parts.hostname, "index.html")
if not os.path.exists(download_path):
success = False
else:
with print_lock:
print("Downloaded:", url)
if not success:
with print_lock:
print("Failed to download:", url)
return False
return True
def DownloadStaticCopies():
global urls
new_urls = []
for url in urls:
url_parts = urlparse(url)
host_dir = os.path.join(output_dir, "data", url_parts.hostname)
download_path = os.path.join(host_dir, url_parts.hostname, "index.html")
if not os.path.exists(download_path):
new_urls.append(url)
if new_urls:
print("Downloading static copies of %d sites..." % len(new_urls))
start_time = time.time()
results = multiprocessing.Pool(20).map(DownloadStaticCopyTask, new_urls)
failed_urls = [new_urls[i] for i,ret in enumerate(results) if not ret]
if failed_urls:
bad_urls_path = os.path.join(output_dir, "data", "bad_urls.txt")
with open(bad_urls_path, 'a') as f:
f.writelines(u + '\n' for u in failed_urls)
failed_urls_set = set(failed_urls)
urls = [u for u in urls if u not in failed_urls_set]
PrintElapsedTime(time.time() - start_time)
SaveWorkingUrls()
def RunDrtTask(url):
url_parts = urlparse(url)
host_dir = os.path.join(output_dir, "data", url_parts.hostname)
html_path = os.path.join(host_dir, url_parts.hostname, "index.html")
if not allow_js:
nojs_path = os.path.join(host_dir, url_parts.hostname, "index-nojs.html")
if not os.path.exists(nojs_path):
with open(html_path) as f:
html = f.read()
if not html:
return False
# These aren't intended to be XSS safe :)
block_tags = (r'<\s*(script|object|video|audio|iframe|frameset|frame)'
r'\b.*?<\s*\/\s*\1\s*>')
block_attrs = r'\s(onload|onerror)\s*=\s*(\'[^\']*\'|"[^"]*|\S*)'
html = re.sub(block_tags, '', html, flags=re.I|re.S)
html = re.sub(block_attrs, '', html, flags=re.I)
with open(nojs_path, 'w') as f:
f.write(html)
html_path = nojs_path
start_time = time.time()
with open(os.devnull, "w") as fnull:
p = subprocess.Popen([content_shell,
"--run-web-tests",
additional_content_shell_flags,
html_path
],
shell=False,
stdout=subprocess.PIPE,
stderr=fnull)
result = p.stdout.read()
PNG_START = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"
PNG_END = b"\x49\x45\x4E\x44\xAE\x42\x60\x82"
try:
start = result.index(PNG_START)
end = result.rindex(PNG_END) + 8
except ValueError:
return False
png_path = os.path.join(output_dir, action, url_parts.hostname + ".png")
MakeDirsIfNotExist(os.path.dirname(png_path))
with open(png_path, 'wb') as f:
f.write(result[start:end])
elapsed_time = (time.time() - start_time, url)
return elapsed_time
def RunDrt():
print("Taking screenshots of %d pages..." % len(urls))
start_time = time.time()
results = multiprocessing.Pool().map(RunDrtTask, urls, 1)
max_time, url = max(t for t in results if t)
elapsed_detail = "(slowest: %.2fs on %s)" % (max_time, url)
PrintElapsedTime(time.time() - start_time, elapsed_detail)
def CompareResultsTask(url):
url_parts = urlparse(url)
before_path = os.path.join(output_dir, "before", url_parts.hostname + ".png")
after_path = os.path.join(output_dir, "after", url_parts.hostname + ".png")
diff_path = os.path.join(output_dir, "diff", url_parts.hostname + ".png")
MakeDirsIfNotExist(os.path.join(output_dir, "diff"))
# TODO(johnme): Don't hardcode "real_world_impact".
red_path = ("data:image/gif;base64,R0lGODlhAQABAPAAAP8AAP///yH5BAAAAAAALAAAAA"
"ABAAEAAAICRAEAOw==")
before_exists = os.path.exists(before_path)
after_exists = os.path.exists(after_path)
if not before_exists and not after_exists:
# TODO(johnme): Make this more informative.
return (-100, url, red_path)
if before_exists != after_exists:
# TODO(johnme): Make this more informative.
return (200, url, red_path)
# Get percentage difference.
p = subprocess.Popen([image_diff, "--histogram",
before_path, after_path],
shell=False,
stdout=subprocess.PIPE)
output,_ = p.communicate()
if p.returncode == 0:
return (0, url, before_path)
diff_match = re.match(r'histogram diff: (\d+\.\d{2})% (?:passed|failed)\n'
'exact diff: (\d+\.\d{2})% (?:passed|failed)', output)
if not diff_match:
raise Exception("image_diff output format changed")
histogram_diff = float(diff_match.group(1))
exact_diff = float(diff_match.group(2))
combined_diff = max(histogram_diff + exact_diff / 8, 0.001)
# Produce diff PNG.
subprocess.call([image_diff, "--diff", before_path, after_path, diff_path])
return (combined_diff, url, diff_path)
def CompareResults():
print("Running image_diff on %d pages..." % len(urls))
start_time = time.time()
results = multiprocessing.Pool().map(CompareResultsTask, urls)
results.sort(key=itemgetter(0), reverse=True)
PrintElapsedTime(time.time() - start_time)
now = datetime.datetime.today().strftime("%a %Y-%m-%d %H:%M")
html_start = textwrap.dedent("""\
<!DOCTYPE html>
<html>
<head>
<title>Real World Impact report %s</title>
<script>
var togglingImg = null;
var toggleTimer = null;
var before = true;
function toggle() {
var newFolder = before ? "before" : "after";
togglingImg.src = togglingImg.src.replace(/before|after|diff/, newFolder);
before = !before;
toggleTimer = setTimeout(toggle, 300);
}
function startToggle(img) {
before = true;
togglingImg = img;
if (!img.origSrc)
img.origSrc = img.src;
toggle();
}
function stopToggle(img) {
clearTimeout(toggleTimer);
img.src = img.origSrc;
}
document.onkeydown = function(e) {
e = e || window.event;
var keyCode = e.keyCode || e.which;
var newFolder;
switch (keyCode) {
case 49: //'1'
newFolder = "before"; break;
case 50: //'2'
newFolder = "after"; break;
case 51: //'3'
newFolder = "diff"; break;
default:
return;
}
var imgs = document.getElementsByTagName("img");
for (var i = 0; i < imgs.length; i++) {
imgs[i].src = imgs[i].src.replace(/before|after|diff/, newFolder);
}
};
</script>
<style>
h1 {
font-family: sans;
}
h2 {
font-family: monospace;
white-space: pre;
}
.nsfw-spacer {
height: 50vh;
}
.nsfw-warning {
background: yellow;
border: 10px solid red;
}
.info {
font-size: 1.2em;
font-style: italic;
}
body:not(.details-supported) details {
display: none;
}
</style>
</head>
<body>
<script>
if ('open' in document.createElement('details'))
document.body.className = "details-supported";
</script>
<!--<div class="nsfw-spacer"></div>-->
<p class="nsfw-warning">Warning: sites below are taken from the Alexa top %d
and may be NSFW.</p>
<!--<div class="nsfw-spacer"></div>-->
<h1>Real World Impact report %s</h1>
<p class="info">Press 1, 2 and 3 to switch between before, after and diff
screenshots respectively; or hover over the images to rapidly alternate
between before and after.</p>
""" % (now, num_sites, now))
html_same_row = """\
<h2>No difference on <a href="%s">%s</a>.</h2>
"""
html_diff_row = """\
<h2>%7.3f%% difference on <a href="%s">%s</a>:</h2>
<img src="%s" width="800" height="600"
onmouseover="startToggle(this)" onmouseout="stopToggle(this)">
"""
html_nsfw_diff_row = """\
<h2>%7.3f%% difference on <a href="%s">%s</a>:</h2>
<details>
<summary>This site may be NSFW. Click to expand/collapse.</summary>
<img src="%s" width="800" height="600"
onmouseover="startToggle(this)" onmouseout="stopToggle(this)">
</details>
"""
html_end = textwrap.dedent("""\
</body>
</html>""")
html_path = os.path.join(output_dir, "diff.html")
with open(html_path, 'w') as f:
f.write(html_start)
for (diff_float, url, diff_path) in results:
diff_path = os.path.relpath(diff_path, output_dir)
if diff_float == 0:
f.write(html_same_row % (url, url))
elif url in nsfw_urls:
f.write(html_nsfw_diff_row % (diff_float, url, url, diff_path))
else:
f.write(html_diff_row % (diff_float, url, url, diff_path))
f.write(html_end)
webbrowser.open_new_tab("file://" + html_path)
def main(argv):
global num_sites, action, allow_js, additional_content_shell_flags
parser = argparse.ArgumentParser(
formatter_class=RawTextHelpFormatter,
description="Compare the real world impact of a content shell change.",
epilog=textwrap.dedent("""\
Example usage:
1. Build content_shell in out/Release without any changes.
2. Run: %s before [num sites to test (default %d)].
3. Either:
a. Apply your controversial patch and rebuild content_shell.
b. Pass --additional_flags="--enable_your_flag" in step 4.
4. Run: %s after [num sites to test (default %d)].
5. Run: %s compare [num sites to test (default %d)].
This will open the results in your web browser.
""" % (argv[0], num_sites, argv[0], num_sites, argv[0], num_sites)))
parser.add_argument("--allow_js", help="Don't disable Javascript",
action="store_true")
parser.add_argument("--additional_flags",
help="Additional flags to pass to content shell")
parser.add_argument("action",
help=textwrap.dedent("""\
Action to perform.
download - Just download the sites.
before - Run content shell and record 'before' result.
after - Run content shell and record 'after' result.
compare - Compare before and after results.
"""),
choices=["download", "before", "after", "compare"])
parser.add_argument("num_sites",
help="Number of sites (default %s)" % num_sites,
type=int, default=num_sites, nargs='?')
args = parser.parse_args()
action = args.action
if (args.num_sites):
num_sites = args.num_sites
if (args.allow_js):
allow_js = args.allow_js
if (args.additional_flags):
additional_content_shell_flags = args.additional_flags
if not SetupPathsAndOut() or not CheckPrerequisites() or not PickSampleUrls():
return 1
if action == 'compare':
CompareResults()
else:
DownloadStaticCopies()
if action != 'download':
RunDrt()
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv))
Codebase Browser
chromium