blob: 3326334e222bb9678119b0b8872a3de6f0621a27 [file] [log] [blame]
#!/usr/bin/env python3.8
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
Example usage:
$ fx set ...
$ scripts/gn/ --issue=-Wshorten-64-to-32 --issue=-Wimplicit-int-float-conversion --config="//build/config:Wno-conversion"
import argparse
import fileinput
import multiprocessing
import os
import re
import subprocess
import json
import sys
import functools
from typing import List, Dict
def run_command(command):
return subprocess.check_output(
command, stderr=subprocess.STDOUT, encoding="utf8")
# functools.cache is more semantically accurate, but requires Python 3.9
def get_out_dir():
"""Retrieve the build output directory"""
fx_status = run_command(["fx", "status", "--format=json"])
return json.loads(
def get_common_gn_args():
"""Retrieve common GN args shared by several commands in this script.
A list of GN command-line arguments (to be used after "gn <command>").
return [get_out_dir()]
def gn_find_refs(files: List[str], raise_if_no_match: bool = True) -> List[str]:
"""Wrapper around `fx gn refs`.
Returns the targets referencing any of the files in `error_files`.
refs_command = ["fx", "gn", "refs"]
refs_command += get_common_gn_args()
refs_command += sorted(files)
refs_out = run_command(refs_command)
except subprocess.CalledProcessError as e:
if ("The input matches no targets, configs, or files." in e.output and
not raise_if_no_match):
return []
print(f"Failed to resolve references for {files}!", file=sys.stderr)
print(e.output, file=sys.stderr)
raise e
return refs_out.splitlines()
def gn_find_refs_complete(
path: str, sources_that_include_header: Dict[str,
List[str]]) -> List[str]:
"""Returns the targets referencing a path.
If no targets refer to that path and the path is a header,
fallback to looking for the targets that reference the source files which
include that header.
If still no targets, return an empty list.
This function is module-level for reasons to do with how Python
multiprocessing works.
params: tuple of (path, dictionary from header to list of sources)
refs = gn_find_refs([path], raise_if_no_match=False)
if refs:
return refs
# If no direct reference, look for refs for the sources instead.
f"Note: looking for source files which includes {path}, "
"because the path was not tracked by the build system",
sources = sources_that_include_header.get(path, [])
if not sources:
f"Warning: {path} was not tracked by the build system, "
"and not included by any sources",
return []
return gn_find_refs(sources)
def can_have_config(params):
"""Returns whether the given target can have a config.
If not sure, returns True.
This function is module-level for reasons to do with how Python
multiprocessing works.
params: tuple of target to examine, config to add.
Packed in a single tuple for reasons to do with how Python
multiprocessing works.
) = params
desc_command = ["fx", "gn", "desc"]
desc_command += get_common_gn_args()
desc_out = run_command(desc_command + [target, "configs"])
# Target has configs and they include the given config.
# Can't add a duplicate config!
return config not in desc_out
except subprocess.CalledProcessError as e:
if 'Don\'t know how to display "configs" for ' in e.output:
# Target type cannot have configs,
# or the actual target is in another toolchain.
# Assume the latter.
return True
elif " matches no targets, configs or files" in e.output:
# The target probably exists in a non-default toolchain.
# Assume that it can have the config.
return True
raise e
included_from_regex = re.compile("In file included from (.*?\.(c|cc|cpp)):\d*")
def find_cc_file_using_included_from(prev_lines: str):
"""Given the last lines of compiler output, find the first path that
is not of the 'In file included from ...' format, which is probably the
source file that led to the error by including a chain of headers.
# The last line should be a header file reference
assert "In file included from" not in prev_lines[
-1], f"unexpected {prev_lines}"
# Go backwards until we hit a '.cc' or '.cpp' file
for l in reversed(prev_lines[:-1]):
match = included_from_regex.match(l)
if match:
return os.path.normpath(os.path.join(get_out_dir(),
raise RuntimeError(f"Cannot find .cc/.cpp file in {prev_lines}")
def main():
parser = argparse.ArgumentParser(
description="Adds a given config to all compile targets with "
"a given compiler error")
help="Captured output from a build invocation. "
"If not provided, the tool will invoke the build first.",
help="If true, pause after each step in the script to review changes",
"If true, attempt to produce a complete annotation by attributing errors "
"in unreferenced headers to source files that include that header.",
help="A regex matching the intended compiler error/warning. "
"E.g. --issue='error:.*-Wconversion'."
"Take care to avoid character expansion by the shell, for example "
"by using single quotation marks.")
parser.add_argument("--config", required=True, help="GN config to add")
parser.add_argument("--comment", help="Comment to add before config")
args = parser.parse_args()
fx_build_log = args.fx_build_log
confirm = args.confirm
complete = args.complete
issue = '|'.join(args.issue)
config = args.config
comment = args.comment
log_regex = re.compile(f"(?:\.\./)*([^:]*):\d*:\d*:.*({issue})")
if fx_build_log:
with open(fx_build_log, "r") as f:
build_out = f.readlines()
# Harvest all compilation errors
# On error continue building to discover all failures
build_out = run_command(["fx", "build", "-k0"])
except subprocess.CalledProcessError as e:
build_out = e.output
build_out = build_out.splitlines()
sources_that_include_header = {}
error_files = set()
prev_lines = []
for line in build_out:
# We probably won't have more than 30 layers of header inclusions.
prev_lines = prev_lines[-30:]
match = log_regex.match(line)
if match:
path = os.path.normpath(
if path.endswith(".h"):
# Trace back to a '.cc' file, and record on the side
sources = sources_that_include_header.get(path, [])
sources_that_include_header[path] = sources
print("Sources with compilation issues:")
if not error_files:
return 0
if confirm:
input("Press Enter to continue...")
# Collect all files with targets referencing failing sources
print("Resolving references...")
target_no_toolchain = re.compile("(\/\/[^:]*:[^(]*)(?:\(.*\))?")
if complete:
# Run `gn ref` on every error file individually. This is the only way
# we can determine if a certain file does not have a reference.
# Cap the max parallelism as the `gn refs` operation turned out very
# memory intensive.
with multiprocessing.Pool(min(multiprocessing.cpu_count(), 20)) as p:
refs_lists = p.starmap(
((path, sources_that_include_header) for path in error_files),
error_targets = {
for refs in refs_lists
for ref in refs
# Use a single `gn refs` invocation to find all references.
# Note that this may silently omit files without references.
refs_out = gn_find_refs(error_files)
error_targets = {
target_no_toolchain.match(ref).group(1) for ref in refs_out
print("Targets with the error:")
if confirm:
input("Press Enter to continue...")
# Remove targets that already have the given config
# or can't have a config in the first place
print("Removing irrelevant targets...")
# `gn desc` is slow so parallelize
# cap the max parallelism as the `gn refs` operation turned out very
# memory intensive.
with multiprocessing.Pool(min(multiprocessing.cpu_count(), 20)) as p:
target_can_have = zip(
((target, config) for target in error_targets)),
error_targets = {
target for target, can_have in target_can_have if can_have
print("Failing targets:")
if confirm:
input("Press Enter to continue...")
# Fix failing targets
ref_regex = re.compile("//([^:]*):([^.(]*).*")
for target in sorted(error_targets):
match = ref_regex.match(target)
build_dir, target_name = match.groups()
target_regex = re.compile(
'^\s*\w*\("' + re.escape(target_name) + '"\) {')
build_file = os.path.join(build_dir, "")
# Format file before processing
run_command(["fx", "format-code", "--files=" + build_file])
print("Fixing", target)
in_target = False
config_printed = False
in_configs = False
curly_brace_depth = 0
secondary_build_file = os.path.join("build", "secondary", build_file)
if os.path.exists(secondary_build_file):
# Sometimes we put third_party files in a shadow directory
# to avoid conflicting with original files.
build_file = secondary_build_file
start_configs_inline = re.compile('configs \+?= \[ "')
start_configs = re.compile("configs \+?= \[")
for line in fileinput.FileInput(build_file, inplace=True):
curly_brace_depth += line.count("{") - line.count("}")
assert curly_brace_depth >= 0
if config_printed:
# We already printed the config, keep running until we exit the target
if curly_brace_depth == target_end_depth:
in_target = False
elif not in_target:
# Try to enter target
in_target = bool(target_regex.match(line))
if in_target:
target_end_depth = curly_brace_depth - 1
elif curly_brace_depth > target_end_depth + 1:
# Ignore inner blocks such as inner definitions and conditionals
elif curly_brace_depth == target_end_depth:
# Last chance to print config before exiting
if comment:
print("#", comment)
print('configs += [ "' + config + '"]')
config_printed = True
in_target = False
elif start_configs_inline.match(line) and config in line:
config_printed = True
elif start_configs_inline.match(line):
line = line[:-3] + ', "' + config + '" ]\n'
config_printed = True
elif start_configs.match(line):
in_configs = True
elif in_configs and line.strip() == '"' + config + '",':
in_configs = False
config_printed = True
elif in_configs and line.strip() == "]":
if comment:
print("#", comment)
print('"' + config + '",')
in_configs = False
config_printed = True
print(line, end="")
if config_printed and not in_target:
# Reset for a possible redefinition of the same target
# (e.g. within another conditional block)
config_printed = False
run_command(["fx", "format-code", "--files=" + build_file])
print("Fixed all of:")
for error_target in sorted(error_targets):
print(' "' + error_target + '",')
return 0
if __name__ == "__main__":