Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env python3
# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Extract source file information from .ninja files."""
# Copied from //tools/binary_size/libsupersize
import argparse
import io
import json
import logging
import os
import re
import sys
sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
import action_helpers
# E.g.:
# build obj/.../foo.o: cxx gen/.../foo.cc || obj/.../foo.inputdeps.stamp
# build obj/.../libfoo.a: alink obj/.../a.o obj/.../b.o |
# build ./libchrome.so ./lib.unstripped/libchrome.so: solink a.o b.o ...
# build libmonochrome.so: __chrome_android_libmonochrome___rule | ...
_REGEX = re.compile(r'build ([^:]+): \w+ (.*?)(?: *\||\n|$)')
_RLIBS_REGEX = re.compile(r' rlibs = (.*?)(?:\n|$)')
# Unmatches seems to happen for empty source_sets(). E.g.:
# obj/chrome/browser/ui/libui_public_dependencies.a
_MAX_UNMATCHED_TO_LOG = 20
_MAX_UNMATCHED_TO_IGNORE = 200
class _SourceMapper:
def __init__(self, dep_map, parsed_files):
self._dep_map = dep_map
self.parsed_files = parsed_files
self._unmatched_paths = set()
def _FindSourceForPathInternal(self, path):
if not path.endswith(')'):
if path.startswith('..'):
return path
return self._dep_map.get(path)
# foo/bar.a(baz.o)
start_idx = path.index('(')
lib_name = path[:start_idx]
obj_name = path[start_idx + 1:-1]
by_basename = self._dep_map.get(lib_name)
if not by_basename:
if lib_name.endswith('rlib') and 'std/' in lib_name:
# Currently we use binary prebuilt static libraries of the Rust
# stdlib so we can't get source paths. That may change in future.
return '(Rust stdlib)/%s' % lib_name
return None
if lib_name.endswith('.rlib'):
# Rust doesn't really have the concept of an object file because
# the compilation unit is the whole 'crate'. Return whichever
# filename was the crate root.
return next(iter(by_basename.values()))
obj_path = by_basename.get(obj_name)
if not obj_path:
# Found the library, but it doesn't list the .o file.
logging.warning('no obj basename for %s %s', path, obj_name)
return None
return self._dep_map.get(obj_path)
def FindSourceForPath(self, path):
"""Returns the source path for the given object path (or None if not found).
Paths for objects within archives should be in the format: foo/bar.a(baz.o)
"""
ret = self._FindSourceForPathInternal(path)
if not ret and path not in self._unmatched_paths:
if self.unmatched_paths_count < _MAX_UNMATCHED_TO_LOG:
logging.warning('Could not find source path for %s (empty source_set?)',
path)
self._unmatched_paths.add(path)
return ret
@property
def unmatched_paths_count(self):
return len(self._unmatched_paths)
def _ParseNinjaPathList(path_list):
ret = path_list.replace('\\ ', '\b')
return [s.replace('\b', ' ') for s in ret.split()]
def _OutputsAreObject(outputs):
return (outputs.endswith('.a') or outputs.endswith('.o')
or outputs.endswith('.rlib'))
def _ParseOneFile(lines, dep_map, executable_path):
sub_ninjas = []
executable_inputs = None
last_executable_paths = []
for line in lines:
if line.startswith('subninja '):
sub_ninjas.append(line[9:-1])
# Rust .rlibs are listed as implicit dependencies of the main
# target linking rule, then are given as an extra
# rlibs =
# variable on a subsequent line. Watch out for that line.
elif m := _RLIBS_REGEX.match(line):
if executable_path in last_executable_paths:
executable_inputs.extend(_ParseNinjaPathList(m.group(1)))
elif m := _REGEX.match(line):
outputs, srcs = m.groups()
if _OutputsAreObject(outputs):
output = outputs.replace('\\ ', ' ')
assert output not in dep_map, 'Duplicate output: ' + output
if output[-1] == 'o':
dep_map[output] = srcs.replace('\\ ', ' ')
else:
obj_paths = _ParseNinjaPathList(srcs)
dep_map[output] = {os.path.basename(p): p for p in obj_paths}
elif executable_path:
last_executable_paths = [
os.path.normpath(p) for p in _ParseNinjaPathList(outputs)
]
if executable_path in last_executable_paths:
executable_inputs = _ParseNinjaPathList(srcs)
return sub_ninjas, executable_inputs
def _Parse(output_directory, executable_path):
"""Parses build.ninja and subninjas.
Args:
output_directory: Where to find the root build.ninja.
executable_path: Path to binary to find inputs for.
Returns: A tuple of (source_mapper, executable_inputs).
source_mapper: _SourceMapper instance.
executable_inputs: List of paths of linker inputs.
"""
if executable_path:
executable_path = os.path.relpath(executable_path, output_directory)
to_parse = ['build.ninja']
seen_paths = set(to_parse)
dep_map = {}
executable_inputs = None
while to_parse:
path = os.path.join(output_directory, to_parse.pop())
with open(path, encoding='utf-8', errors='ignore') as obj:
sub_ninjas, found_executable_inputs = _ParseOneFile(
obj, dep_map, executable_path)
if found_executable_inputs:
assert not executable_inputs, (
'Found multiple inputs for executable_path ' + executable_path)
executable_inputs = found_executable_inputs
for subpath in sub_ninjas:
assert subpath not in seen_paths, 'Double include of ' + subpath
seen_paths.add(subpath)
to_parse.extend(sub_ninjas)
assert executable_inputs, 'Failed to find rule that builds ' + executable_path
return _SourceMapper(dep_map, seen_paths), executable_inputs
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--executable', required=True)
parser.add_argument('--result-json', required=True)
parser.add_argument('--depfile')
args = parser.parse_args()
logs_io = io.StringIO()
logging.basicConfig(level=logging.DEBUG,
format='%(levelname).1s %(relativeCreated)6d %(message)s',
stream=logs_io)
source_mapper, object_paths = _Parse('.', args.executable)
logging.info('Found %d linker inputs', len(object_paths))
source_paths = []
for obj_path in object_paths:
result = source_mapper.FindSourceForPath(obj_path) or obj_path
# Need to recurse on .a files.
if isinstance(result, dict):
source_paths.extend(
source_mapper.FindSourceForPath(v) or v for v in result.values())
else:
source_paths.append(result)
logging.info('Found %d source paths', len(source_paths))
num_unmatched = source_mapper.unmatched_paths_count
if num_unmatched > _MAX_UNMATCHED_TO_LOG:
logging.warning('%d paths were missing sources (showed the first %d)',
num_unmatched, _MAX_UNMATCHED_TO_LOG)
if num_unmatched > _MAX_UNMATCHED_TO_IGNORE:
raise Exception('Too many unmapped files. Likely a bug in ninja_parser.py')
if args.depfile:
action_helpers.write_depfile(args.depfile, args.result_json,
source_mapper.parsed_files)
with open(args.result_json, 'w', encoding='utf-8') as f:
json.dump({
'logs': logs_io.getvalue(),
'source_paths': source_paths,
}, f)
if __name__ == '__main__':
main()