#!/usr/bin/env python3
# Copyright 2013 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A tool to generate symbols for a binary suitable for breakpad.
Currently, the tool only supports Linux, Android, and Mac. Support for other
platforms is planned.
"""
import argparse
import collections
import errno
import glob
import multiprocessing
import os
import queue
import re
import shutil
import subprocess
import sys
import threading
import traceback
CONCURRENT_TASKS = multiprocessing.cpu_count()
if sys.platform == 'win32':
# TODO(crbug.com/40755900) - we can't use more than 56
# cores on Windows or Python3 may hang.
CONCURRENT_TASKS = min(CONCURRENT_TASKS, 56)
# The BINARY_INFO tuple describes a binary as dump_syms identifies it.
BINARY_INFO = collections.namedtuple('BINARY_INFO',
['platform', 'arch', 'hash', 'name'])
def _GetDumpSymsBinary(dump_syms_path: str, build_dir: str):
"""Returns the path to the dump_syms binary."""
if not dump_syms_path:
DUMP_SYMS = 'dump_syms'
dump_syms_path = os.path.join(os.path.expanduser(build_dir), DUMP_SYMS)
if not os.access(dump_syms_path, os.X_OK):
print(f'Cannot find {dump_syms_path}.')
return None
return dump_syms_path
def Resolve(path, exe_path, loader_path, rpaths):
"""Resolve a dyld path.
@executable_path is replaced with |exe_path|
@loader_path is replaced with |loader_path|
@rpath is replaced with the first path in |rpaths| where the referenced file
is found
"""
path = path.replace('@loader_path', loader_path)
path = path.replace('@executable_path', exe_path)
if path.find('@rpath') != -1:
for rpath in rpaths:
new_path = path.replace('@rpath', rpath)
if os.access(new_path, os.X_OK):
return new_path
return ''
return path
def GetSharedLibraryDependenciesLinux(binary):
"""Return absolute paths to all shared library dependencies of the binary.
This implementation assumes that we're running on a Linux system."""
ldd = subprocess.check_output(['ldd', binary]).decode('utf-8')
lib_re = re.compile('\t.* => (.+) \(.*\)$')
result = []
for line in ldd.splitlines():
m = lib_re.match(line)
if m:
result.append(os.path.abspath(m.group(1)))
return result
def _GetSharedLibraryDependenciesAndroidOrChromeOS(binary):
"""GetSharedLibraryDependencies* suitable for Android or ChromeOS.
Both assume that the host is Linux-based, but the binary being symbolized is
being run on a device with potentially different architectures. Unlike ldd,
readelf plays nice with mixed host/device architectures (e.g. x86-64 host,
arm64 device), so use that.
"""
readelf = subprocess.check_output(['readelf', '-d', binary]).decode('utf-8')
lib_re = re.compile('Shared library: \[(.+)\]$')
result = []
binary_path = os.path.dirname(os.path.abspath(binary))
for line in readelf.splitlines():
m = lib_re.search(line)
if m:
lib = os.path.join(binary_path, m.group(1))
if os.access(lib, os.X_OK):
result.append(lib)
return result
def GetSharedLibraryDependenciesAndroid(binary):
"""Return absolute paths to all shared library dependencies of the binary.
This implementation assumes that we're running on a Linux system, but
compiled for Android."""
return _GetSharedLibraryDependenciesAndroidOrChromeOS(binary)
def GetDeveloperDirMac():
"""Finds a good DEVELOPER_DIR value to run Mac dev tools.
It checks the existing DEVELOPER_DIR and `xcode-select -p` and uses
one of those if the folder exists, and falls back to one of the
existing system folders with dev tools.
Returns:
(string) path to assign to DEVELOPER_DIR env var.
"""
candidate_paths = []
if 'DEVELOPER_DIR' in os.environ:
candidate_paths.append(os.environ['DEVELOPER_DIR'])
candidate_paths.extend([
subprocess.check_output(['xcode-select', '-p']).decode('utf-8').strip(),
# Most Mac 10.1[0-2] bots have at least one Xcode installed.
'/Applications/Xcode.app',
'/Applications/Xcode9.0.app',
'/Applications/Xcode8.0.app',
# Mac 10.13 bots don't have any Xcode installed, but have CLI tools as a
# temporary workaround.
'/Library/Developer/CommandLineTools',
])
for path in candidate_paths:
if os.path.exists(path):
return path
print('WARNING: no value found for DEVELOPER_DIR. Some commands may fail.')
def GetSharedLibraryDependenciesMac(binary, exe_path):
"""Return absolute paths to all shared library dependencies of the binary.
This implementation assumes that we're running on a Mac system."""
# realpath() serves two purposes:
# 1. If an executable is linked against a framework, it links against
# Framework.framework/Framework, which is a symlink to
# Framework.framework/Framework/Versions/A/Framework. rpaths are relative
# to the real location, so resolving the symlink is important.
# 2. It converts binary to an absolute path. If binary is just
# "foo.dylib" in the current directory, dirname() would return an empty
# string, causing "@loader_path/foo" to incorrectly expand to "/foo".
loader_path = os.path.dirname(os.path.realpath(binary))
env = os.environ.copy()
SRC_ROOT_PATH = os.path.join(os.path.dirname(__file__), '../../../..')
hermetic_otool_path = os.path.join(SRC_ROOT_PATH, 'build', 'mac_files',
'xcode_binaries', 'Contents', 'Developer',
'Toolchains', 'XcodeDefault.xctoolchain',
'usr', 'bin', 'otool')
if os.path.exists(hermetic_otool_path):
otool_path = hermetic_otool_path
else:
developer_dir = GetDeveloperDirMac()
if developer_dir:
env['DEVELOPER_DIR'] = developer_dir
otool_path = 'otool'
otool = subprocess.check_output([otool_path, '-lm', binary],
env=env).decode('utf-8').splitlines()
rpaths = []
dylib_id = None
for idx, line in enumerate(otool):
if line.find('cmd LC_RPATH') != -1:
m = re.match(' *path (.*) \(offset .*\)$', otool[idx + 2])
rpath = m.group(1)
rpath = rpath.replace('@loader_path', loader_path)
rpath = rpath.replace('@executable_path', exe_path)
rpaths.append(rpath)
elif line.find('cmd LC_ID_DYLIB') != -1:
m = re.match(' *name (.*) \(offset .*\)$', otool[idx + 2])
dylib_id = m.group(1)
# `man dyld` says that @rpath is resolved against a stack of LC_RPATHs from
# all executable images leading to the load of the current module. This is
# intentionally not implemented here, since we require that every .dylib
# contains all the rpaths it needs on its own, without relying on rpaths of
# the loading executables.
otool = subprocess.check_output([otool_path, '-Lm', binary],
env=env).decode('utf-8').splitlines()
lib_re = re.compile('\t(.*) \(compatibility .*\)$')
deps = []
for line in otool:
m = lib_re.match(line)
if m:
# For frameworks and shared libraries, `otool -L` prints the LC_ID_DYLIB
# as the first line. Filter that out.
if m.group(1) == dylib_id:
continue
dep = Resolve(m.group(1), exe_path, loader_path, rpaths)
if dep:
deps.append(os.path.normpath(dep))
else:
print(('ERROR: failed to resolve %s, exe_path %s, loader_path %s, '
'rpaths %s' %
(m.group(1), exe_path, loader_path, ', '.join(rpaths))),
file=sys.stderr)
sys.exit(1)
return deps
def GetSharedLibraryDependenciesChromeOS(binary):
"""Return absolute paths to all shared library dependencies of the binary.
This implementation assumes that we're running on a Linux system, but
compiled for ChromeOS."""
return _GetSharedLibraryDependenciesAndroidOrChromeOS(binary)
def GetSharedLibraryDependencies(options, binary, exe_path):
"""Return absolute paths to all shared library dependencies of the binary."""
deps = []
if options.platform == 'linux':
deps = GetSharedLibraryDependenciesLinux(binary)
elif options.platform == 'android':
deps = GetSharedLibraryDependenciesAndroid(binary)
elif options.platform == 'darwin':
deps = GetSharedLibraryDependenciesMac(binary, exe_path)
elif options.platform == 'chromeos':
deps = GetSharedLibraryDependenciesChromeOS(binary)
else:
print("Platform not supported.")
sys.exit(1)
result = []
build_dir = os.path.abspath(options.build_dir)
for dep in deps:
if (os.access(dep, os.X_OK)
and os.path.abspath(os.path.dirname(dep)).startswith(build_dir)):
result.append(dep)
return result
def GetTransitiveDependencies(options):
"""Return absolute paths to the transitive closure of all shared library
dependencies of the binary, along with the binary itself."""
binary = os.path.abspath(options.binary)
exe_path = os.path.dirname(binary)
if options.platform == 'linux':
# 'ldd' returns all transitive dependencies for us.
deps = set(GetSharedLibraryDependencies(options, binary, exe_path))
deps.add(binary)
return list(deps)
elif (options.platform == 'darwin' or options.platform == 'android'
or options.platform == 'chromeos'):
binaries = set([binary])
q = [binary]
while q:
deps = GetSharedLibraryDependencies(options, q.pop(0), exe_path)
new_deps = set(deps) - binaries
binaries |= new_deps
q.extend(list(new_deps))
return binaries
print("Platform not supported.")
sys.exit(1)
def mkdir_p(path):
"""Simulates mkdir -p."""
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def GetBinaryInfoFromHeaderInfo(header_info):
"""Given a standard symbol header information line, returns BINARY_INFO."""
# header info is of the form "MODULE $PLATFORM $ARCH $HASH $BINARY"
info_split = header_info.strip().split(' ', 4)
if len(info_split) != 5 or info_split[0] != 'MODULE':
return None
return BINARY_INFO(*info_split[1:])
def CreateSymbolDir(options, output_dir, relative_hash_dir):
"""Create the directory to store breakpad symbols in. On Android/Linux, we
also create a symlink in case the hash in the binary is missing."""
mkdir_p(output_dir)
if options.platform == 'android' or options.platform == 'linux':
try:
os.symlink(
relative_hash_dir,
os.path.join(os.path.dirname(output_dir),
'000000000000000000000000000000000'))
except:
pass
def GenerateSymbols(options, binaries):
"""Dumps the symbols of binary and places them in the given directory."""
q = queue.Queue()
exceptions = []
print_lock = threading.Lock()
exceptions_lock = threading.Lock()
def _Worker():
dump_syms = _GetDumpSymsBinary(options.dump_syms_path, options.build_dir)
while True:
try:
should_dump_syms = True
reason = "no reason"
binary = q.get()
run_once = True
while run_once:
run_once = False
if not dump_syms:
should_dump_syms = False
reason = "Could not locate dump_syms executable."
break
dump_syms_output = subprocess.check_output([dump_syms, '-i',
binary]).decode('utf-8')
header_info = dump_syms_output.splitlines()[0]
binary_info = GetBinaryInfoFromHeaderInfo(header_info)
if not binary_info:
should_dump_syms = False
reason = "Could not obtain binary information."
break
# See if the output file already exists.
output_dir = os.path.join(options.symbols_dir, binary_info.name,
binary_info.hash)
output_path = os.path.join(output_dir, binary_info.name + '.sym')
if os.path.isfile(output_path):
should_dump_syms = False
reason = "Symbol file already found."
break
# See if there is a symbol file already found next to the binary
potential_symbol_files = glob.glob('%s.breakpad*' % binary)
for potential_symbol_file in potential_symbol_files:
with open(potential_symbol_file, 'rt') as f:
symbol_info = GetBinaryInfoFromHeaderInfo(f.readline())
if symbol_info == binary_info:
CreateSymbolDir(options, output_dir, binary_info.hash)
shutil.copyfile(potential_symbol_file, output_path)
should_dump_syms = False
reason = "Found local symbol file."
break
if not should_dump_syms:
if options.verbose:
with print_lock:
print("Skipping %s (%s)" % (binary, reason))
continue
if options.verbose:
with print_lock:
print("Generating symbols for %s" % binary)
CreateSymbolDir(options, output_dir, binary_info.hash)
with open(output_path, 'wb') as f:
subprocess.check_call([dump_syms, '-r', binary], stdout=f)
except Exception as e:
with exceptions_lock:
exceptions.append(traceback.format_exc())
finally:
q.task_done()
for binary in binaries:
q.put(binary)
for _ in range(options.jobs):
t = threading.Thread(target=_Worker)
t.daemon = True
t.start()
q.join()
if exceptions:
exception_str = ('One or more exceptions occurred while generating '
'symbols:\n')
exception_str += '\n'.join(exceptions)
raise Exception(exception_str)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build-dir',
required=True,
help='The build output directory.')
parser.add_argument('--symbols-dir',
required=True,
help='The directory where to write the symbols file.')
parser.add_argument('--binary',
required=True,
help='The path of the binary to generate symbols for.')
parser.add_argument('--dump-syms-path',
default='',
help='The path of the dump_syms binary. If not provided, '
'by default looks in --build-dir for it.')
parser.add_argument('--clear',
default=False,
action='store_true',
help='Clear the symbols directory before writing new '
'symbols.')
parser.add_argument('-j',
'--jobs',
default=CONCURRENT_TASKS,
action='store',
type=int,
help='Number of parallel tasks to run.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Print verbose status output.')
parser.add_argument('--platform',
default=sys.platform,
help='Target platform of the binary.')
args = parser.parse_args()
if not os.access(args.binary, os.X_OK):
print(f'Cannot find {args.binary}')
return 1
if args.clear:
try:
shutil.rmtree(args.symbols_dir)
except:
pass
if not _GetDumpSymsBinary(args.dump_syms_path, args.build_dir):
return 1
# Build the transitive closure of all dependencies.
binaries = GetTransitiveDependencies(args)
GenerateSymbols(args, binaries)
return 0
if '__main__' == __name__:
sys.exit(main())
Codebase Browser
chromium