#!/usr/bin/env python3
# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Runs nm on specified .a and .o file, plus some analysis.
CollectAliasesByAddress():
Runs nm on the elf to collect all symbol names. This reveals symbol names of
identical-code-folded functions.
CollectAliasesByAddressAsync():
Runs CollectAliasesByAddress in a subprocess and returns a promise.
RunNmOnIntermediates():
BulkForkAndCall() target: Runs nm on a .a file or a list of .o files, parses
the output, extracts symbol information, and (if available) extracts string
offset information.
CreateUniqueSymbols():
Creates Symbol objects from nm output.
"""
import argparse
import collections
import logging
import os
import subprocess
import demangle
import models
import parallel
import path_util
import readelf
import sys
def _IsRelevantNmName(name):
# Skip lines like:
# 00000000 t $t
# 00000000 r $d.23
# 00000344 N
return name and not name.startswith('$')
def _IsRelevantObjectFileName(name):
# Prevent marking compiler-generated symbols as candidates for shared paths.
# E.g., multiple files might have "CSWTCH.12", but they are different symbols.
#
# Find these via:
# size_info.symbols.GroupedByFullName(min_count=-2).Filter(
# lambda s: s.WhereObjectPathMatches('{')).SortedByCount()
# and then search for {shared}.
# List of names this applies to:
# startup
# __tcf_0 <-- Generated for global destructors.
# ._79
# .Lswitch.table, .Lswitch.table.12
# CSWTCH.12
# lock.12
# table.12
# __compound_literal.12
# .L.ref.tmp.1
# .L.str, .L.str.3
# .L__func__.main: (when using __func__)
# .L__FUNCTION__._ZN6webrtc17AudioDeviceBuffer11StopPlayoutEv
# .L__PRETTY_FUNCTION__._Unwind_Resume
# .L_ZZ24ScaleARGBFilterCols_NEONE9dx_offset (an array literal)
if name in ('__tcf_0', 'startup'):
return False
if name.startswith('._') and name[2:].isdigit():
return False
if name.startswith('.L') and name.find('.', 2) != -1:
return False
dot_idx = name.find('.')
if dot_idx == -1:
return True
name = name[:dot_idx]
return name not in ('CSWTCH', 'lock', '__compound_literal', 'table')
def CollectAliasesByAddress(elf_path):
"""Runs nm on |elf_path| and returns a dict of address->[names]"""
# Constructors often show up twice, so use sets to ensure no duplicates.
names_by_address = collections.defaultdict(set)
# Many OUTLINED_FUNCTION_* entries can coexist on a single address, possibly
# mixed with regular symbols. However, naively keeping these is bad because:
# * OUTLINED_FUNCTION_* can have many duplicates. Keeping them would cause
# false associations downstream, when looking up object_paths from names.
# * For addresses with multiple OUTLINED_FUNCTION_* entries, we can't get the
# associated object_path (exception: the one entry in the .map file, for LLD
# without ThinLTO). So keeping copies around is rather useless.
# Our solution is to merge OUTLINED_FUNCTION_* entries at the same address
# into a single symbol. We'd also like to keep track of the number of copies
# (although it will not be used to compute PSS computation). This is done by
# writing the count in the name, e.g., '** outlined function * 5'.
num_outlined_functions_at_address = collections.Counter()
# About 60mb of output, but piping takes ~30s, and loading it into RAM
# directly takes 3s.
args = [path_util.GetNmPath(), '--no-sort', '--defined-only', elf_path]
# pylint: disable=unexpected-keyword-arg
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
encoding='utf-8')
# llvm-nm may write to stderr. Discard to denoise.
stdout, _ = proc.communicate()
assert proc.returncode == 0
for line in stdout.splitlines():
space_idx = line.find(' ')
address_str = line[:space_idx]
section = line[space_idx + 1]
mangled_name = line[space_idx + 3:]
# To verify that rodata does not have aliases:
# nm --no-sort --defined-only libchrome.so > nm.out
# grep -v '\$' nm.out | grep ' r ' | sort | cut -d' ' -f1 > addrs
# wc -l < addrs; uniq < addrs | wc -l
if section not in 'tTW' or not _IsRelevantNmName(mangled_name):
continue
address = int(address_str, 16)
if not address:
continue
if mangled_name.startswith('OUTLINED_FUNCTION_'):
num_outlined_functions_at_address[address] += 1
else:
names_by_address[address].add(mangled_name)
# Need to add before demangling because |names_by_address| changes type.
for address, count in num_outlined_functions_at_address.items():
name = '** outlined function' + (' * %d' % count if count > 1 else '')
names_by_address[address].add(name)
# Demangle all names.
demangle.DemangleSetsInDictsInPlace(names_by_address)
# Since this is run in a separate process, minimize data passing by returning
# only aliased symbols.
# Also: Sort to ensure stable ordering.
return {
addr: sorted(names, key=lambda n: (n.startswith('**'), n))
for addr, names in names_by_address.items()
if len(names) > 1 or num_outlined_functions_at_address.get(addr, 0) > 1
}
def CreateUniqueSymbols(elf_path, section_ranges):
"""Creates symbols from nm --print-size output.
Creates only one symbol for each address (does not create symbol aliases).
"""
# Filter to sections we care about and sort by (address, size).
section_ranges = [
x for x in section_ranges.items() if x[0] in models.NATIVE_SECTIONS
]
section_ranges.sort(key=lambda x: x[1])
min_address = section_ranges[0][1][0]
max_address = sum(section_ranges[-1][1])
args = [
path_util.GetNmPath(), '--no-sort', '--defined-only', '--print-size',
elf_path
]
# pylint: disable=unexpected-keyword-arg
stdout = subprocess.check_output(args,
stderr=subprocess.DEVNULL,
encoding='utf-8')
lines = stdout.splitlines()
logging.debug('Parsing %d lines of output', len(lines))
symbols_by_address = {}
# Example 32-bit output:
# 00857f94 00000004 t __on_dlclose_late
# 000001ec r ndk_build_number
for line in lines:
tokens = line.split(' ', 3)
num_tokens = len(tokens)
if num_tokens < 3:
# Address with no size and no name.
continue
address_str = tokens[0]
# Check if size is omitted (can happen with binutils but not llvm).
if num_tokens == 3:
size_str = '0'
section = tokens[1]
mangled_name = tokens[2]
else:
size_str = tokens[1]
section = tokens[2]
mangled_name = tokens[3]
if section not in 'BbDdTtRrWw' or not _IsRelevantNmName(mangled_name):
continue
address = int(address_str, 16)
# Ignore symbols outside of sections that we care about.
# Symbols can still exist in sections that we do not care about if those
# sections are interleaved. We discard such symbols in the next loop.
if not min_address <= address < max_address:
continue
# Pick the alias that defines a size.
existing_alias = symbols_by_address.get(address)
if existing_alias and existing_alias.size > 0:
continue
size = int(size_str, 16)
# E.g.: .str.2.llvm.12282370934750212
if mangled_name.startswith('.str.'):
mangled_name = models.STRING_LITERAL_NAME
elif mangled_name.startswith('__ARMV7PILongThunk_'):
# Convert thunks from prefix to suffix so that name is demangleable.
mangled_name = mangled_name[len('__ARMV7PILongThunk_'):] + '.LongThunk'
elif mangled_name.startswith('__ThumbV7PILongThunk_'):
mangled_name = mangled_name[len('__ThumbV7PILongThunk_'):] + '.LongThunk'
# Use address (next loop) to determine between .data and .data.rel.ro.
section_name = None
if section in 'Tt':
section_name = models.SECTION_TEXT
elif section in 'Rr':
section_name = models.SECTION_RODATA
elif section in 'Bb':
section_name = models.SECTION_BSS
# No need to demangle names since they will be demangled by
# DemangleRemainingSymbols().
symbols_by_address[address] = models.Symbol(section_name,
size,
address=address,
full_name=mangled_name)
logging.debug('Sorting %d NM symbols', len(symbols_by_address))
# Sort symbols by address.
sorted_symbols = sorted(symbols_by_address.values(), key=lambda s: s.address)
# Assign section to symbols based on address, and size where unspecified.
# Use address rather than nm's section character to distinguish between
# .data.rel.ro and .data.
logging.debug('Assigning section_name and filling in missing sizes')
section_range_iter = iter(section_ranges)
section_end = -1
raw_symbols = []
active_assembly_sym = None
for i, sym in enumerate(sorted_symbols):
# Move to next section if applicable.
while sym.address >= section_end:
section_range = next(section_range_iter)
section_name, (section_start, section_size) = section_range
section_end = section_start + section_size
# Skip symbols that don't fall into a section that we care about
# (e.g. GCC_except_table533 from .eh_frame).
if sym.address < section_start:
continue
if sym.section_name and sym.section_name != section_name:
logging.warning('Re-assigning section for %r to %s', sym, section_name)
sym.section_name = section_name
if i + 1 < len(sorted_symbols):
next_addr = min(section_end, sorted_symbols[i + 1].address)
else:
next_addr = section_end
# Heuristic: Discard subsequent assembly symbols (no size) that are ALL_CAPS
# or .-prefixed, since they are likely labels within a function.
if (active_assembly_sym and sym.size == 0
and sym.section_name == models.SECTION_TEXT):
if sym.full_name.startswith('.') or sym.full_name.isupper():
active_assembly_sym.size += next_addr - sym.address
# Triggers ~30 times for all of libchrome.so.
logging.debug('Discarding assembly label: %s', sym.full_name)
continue
active_assembly_sym = sym if sym.size == 0 else None
# For assembly symbols:
# Add in a size when absent and guard against size overlapping next symbol.
if active_assembly_sym or sym.end_address > next_addr:
sym.size = next_addr - sym.address
raw_symbols.append(sym)
return raw_symbols
def _CollectAliasesByAddressAsyncHelper(elf_path):
result = CollectAliasesByAddress(elf_path)
return parallel.EncodeDictOfLists(result, key_transform=str)
def CollectAliasesByAddressAsync(elf_path):
"""Calls CollectAliasesByAddress in a helper process. Returns a Result."""
def decode(encoded):
return parallel.DecodeDictOfLists(encoded, key_transform=int)
return parallel.ForkAndCall(_CollectAliasesByAddressAsyncHelper, (elf_path, ),
decode_func=decode)
def _ParseOneObjectFileNmOutput(lines):
# Constructors are often repeated because they have the same unmangled
# name, but multiple mangled names. See:
# https://stackoverflow.com/questions/6921295/dual-emission-of-constructor-symbols
symbol_names = set()
string_addresses = []
for line in lines:
if not line:
break
space_idx = line.find(' ') # Skip over address.
section = line[space_idx + 1]
mangled_name = line[space_idx + 3:]
if _IsRelevantNmName(mangled_name):
# Refer to _IsRelevantObjectFileName() for examples of names.
if section == 'r' and (
mangled_name.startswith('.L.str') or
mangled_name.startswith('.L__') and mangled_name.find('.', 3) != -1):
# Leave as a string for easier marshalling.
string_addresses.append(line[:space_idx].lstrip('0') or '0')
elif _IsRelevantObjectFileName(mangled_name):
symbol_names.add(mangled_name)
return symbol_names, string_addresses
# This is a target for BulkForkAndCall().
def RunNmOnIntermediates(target, output_directory):
"""Returns encoded_symbol_names_by_path, encoded_string_addresses_by_path.
Args:
target: Either a single path to a .a (as a string), or a list of .o paths.
"""
is_archive = isinstance(target, str)
args = [path_util.GetNmPath(), '--no-sort', '--defined-only']
if is_archive:
args.append(target)
else:
args.extend(target)
proc = subprocess.Popen(
args,
cwd=output_directory,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding='utf-8')
# llvm-nm can print 'no symbols' to stderr. Capture and count the number of
# lines, to be returned to the caller.
stdout, stderr = proc.communicate()
assert proc.returncode == 0, 'NM failed: ' + ' '.join(args)
num_no_symbols = len(stderr.splitlines())
lines = stdout.splitlines()
# Empty .a file has no output.
if not lines:
return parallel.EMPTY_ENCODED_DICT, parallel.EMPTY_ENCODED_DICT
is_multi_file = not lines[0]
lines = iter(lines)
if is_multi_file:
next(lines)
path = next(lines)[:-1] # Path ends with a colon.
else:
assert not is_archive
path = target[0]
symbol_names_by_path = {}
string_addresses_by_path = {}
while path:
if is_archive:
# E.g. foo/bar.a(baz.o)
path = '%s(%s)' % (target, path)
mangled_symbol_names, string_addresses = _ParseOneObjectFileNmOutput(lines)
symbol_names_by_path[path] = mangled_symbol_names
if string_addresses:
string_addresses_by_path[path] = string_addresses
path = next(lines, ':')[:-1]
# The multiprocess API uses pickle, which is ridiculously slow. More than 2x
# faster to use join & split.
# TODO(agrieve): We could use path indices as keys rather than paths to cut
# down on marshalling overhead.
return (parallel.EncodeDictOfLists(symbol_names_by_path),
parallel.EncodeDictOfLists(string_addresses_by_path), num_no_symbols)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output-directory', required=True)
parser.add_argument('elf_path', type=os.path.realpath)
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG,
format='%(levelname).1s %(relativeCreated)6d %(message)s')
# Other functions in this file have test entrypoints in object_analyzer.py.
section_ranges = readelf.SectionInfoFromElf(args.elf_path)
symbols = CreateUniqueSymbols(args.elf_path, section_ranges)
for s in symbols:
print(s)
logging.warning('Printed %d symbols', len(symbols))
if __name__ == '__main__':
main()
Codebase Browser
chromium