blob: 239ab34e1ac1b4a91fc91979965b67817e60c0f8 [file]
# Copyright 2025 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Process the debug_symbols.json build API module in various ways."""
import json
import os
import shutil
import subprocess
import time
import typing as T
from pathlib import Path
DebugSymbolEntryType = dict[str, T.Any]
# Binaries whose file name appears in this set are ignored.
# See https://fxbug.dev/445670194 for context. This can be
# removed once https://fxbug.dev/445786243 is fixed.
_DEBUG_BINARY_DENYLIST = {
"static-pie-noop-test",
"static-pie-custom-startup-test",
}
def _find_build_id_in_notes(data: bytes, align: int = 4) -> str:
"""Finds the build-id value in a sequence of ELF note structures."""
offset = 0
total = len(data)
align_mask = align - 1
while offset + 12 <= total:
namesz = int.from_bytes(data[offset : offset + 4], "little")
descsz = int.from_bytes(data[offset + 4 : offset + 8], "little")
ntype = int.from_bytes(data[offset + 8 : offset + 12], "little")
name_offset = offset + 12
name_aligned = (namesz + align_mask) & ~align_mask
desc_offset = name_offset + name_aligned
desc_aligned = (descsz + align_mask) & ~align_mask
if desc_offset + descsz > total or namesz > 256 or descsz > 256:
break
name = data[name_offset : name_offset + namesz]
if ntype == 3 and name.rstrip(b"\0") == b"GNU":
return data[desc_offset : desc_offset + descsz].hex()
offset = desc_offset + desc_aligned
return ""
def extract_gnu_build_id(elf_file: str | Path) -> str:
"""Extracts the GNU build ID from an ELF file (ELF64 or ELF32).
Args:
elf_file: Path to input file.
Returns:
The build-id value as an hexadecimal string, or
an empty string on failure (e.g. not an ELF file,
or no .note.gnu.build-id in it).
"""
try:
# See https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
with open(elf_file, "rb") as f:
# Read ELF header
e_ident = f.read(16)
if len(e_ident) < 16 or e_ident[:4] != b"\x7fELF":
return "" # Not an ELF file
ei_class = e_ident[4]
if ei_class not in (1, 2):
return "" # Not ELF32 or ELF64
is_64 = ei_class == 2
ei_data = e_ident[5]
if ei_data != 1: # ELFDATA2LSB (little-endian)
return ""
def read_int(offset: int, size: int) -> int:
f.seek(offset)
return int.from_bytes(f.read(size), "little")
if is_64:
e_phoff = read_int(32, 8)
e_shoff = read_int(40, 8)
e_phentsize = read_int(54, 2)
e_phnum = read_int(56, 2)
e_shentsize = read_int(58, 2)
e_shnum = read_int(60, 2)
e_shstrndx = read_int(62, 2)
else:
e_phoff = read_int(28, 4)
e_shoff = read_int(32, 4)
e_phentsize = read_int(42, 2)
e_phnum = read_int(44, 2)
e_shentsize = read_int(46, 2)
e_shnum = read_int(48, 2)
e_shstrndx = read_int(50, 2)
# 1. Check section headers if present.
if e_shoff != 0 and e_shnum > 0 and e_shstrndx < e_shnum:
shstrtab_entry = e_shoff + e_shentsize * e_shstrndx
shstrtab_offset = (
read_int(shstrtab_entry + 24, 8)
if is_64
else read_int(shstrtab_entry + 16, 4)
)
for i in range(e_shnum):
entry = e_shoff + i * e_shentsize
sh_name = read_int(entry, 4)
if is_64:
sh_offset = read_int(entry + 24, 8)
sh_size = read_int(entry + 32, 8)
sh_addralign = read_int(entry + 48, 8)
else:
sh_offset = read_int(entry + 16, 4)
sh_size = read_int(entry + 20, 4)
sh_addralign = read_int(entry + 32, 4)
f.seek(shstrtab_offset + sh_name)
name = b""
while True:
byte = f.read(1)
if not byte or byte == b"\x00":
break
name += byte
if name == b".note.gnu.build-id":
f.seek(sh_offset)
data = f.read(sh_size)
align = sh_addralign if sh_addralign in (4, 8) else 4
build_id = _find_build_id_in_notes(data, align=align)
if build_id:
return build_id
if sh_size > 16:
return data[16:].hex()
# 2. Check program headers for PT_NOTE (type 4).
if e_phoff != 0 and e_phnum > 0:
for i in range(e_phnum):
entry = e_phoff + i * e_phentsize
p_type = read_int(entry, 4)
if p_type == 4: # PT_NOTE
if is_64:
p_offset = read_int(entry + 8, 8)
p_filesz = read_int(entry + 32, 8)
p_align = read_int(entry + 48, 8)
else:
p_offset = read_int(entry + 4, 4)
p_filesz = read_int(entry + 16, 4)
p_align = read_int(entry + 28, 4)
if p_filesz > 0:
f.seek(p_offset)
data = f.read(p_filesz)
align = p_align if p_align in (4, 8) else 4
build_id = _find_build_id_in_notes(
data, align=align
)
if build_id:
return build_id
except Exception:
# Ignore missing file, unreadable file, or truncated file errors.
pass
return ""
class DebugSymbolsManifestParser(object):
"""Parse the debug_symbols.json build API module and expand its
included manifests to generate a flattened list of entries.
See comments for //:debug_symbols target for schema description.
Usage is:
1) Create instance.
2) Call parse_manifest_file(). This will raise ValueError
if the input is malformed, if an include cycle is
detected.
3) Use the |entries| property to get the flatenned list of entries,
with duplicates removed.
4) Use the |extra_input_files| property to get the files read by the
parser due to manifest includes or when reading the build-ID values.
"""
def __init__(self, build_dir: None | Path | str = None) -> None:
"""Create instance.
Args:
build_dir: Optional build directory Path. For entries that have a
"manifest" key, the value will be a path relative to this value.
Default value is the current directory.
"""
self._build_dir = Path(build_dir) if build_dir else Path.cwd()
self._visited_paths: set[Path] = set()
self._visited_stack: list[Path] = []
self._entries: list[DebugSymbolEntryType] = []
self._resolve_build_id = False
self._get_build_id: T.Callable[[str | Path], str] = extract_gnu_build_id
self._elfinfo: T.Any = None
def enable_build_id_resolution(self) -> None:
"""Ensure the build-id value is computed if not present in an entry."""
self._resolve_build_id = True
def set_build_id_callback_for_test(
self, get_build_id: T.Callable[[str | Path], str]
) -> None:
"""Override the function used to compute the BuildID, only used for tests."""
self._get_build_id = get_build_id
def extract_build_id(self, path: str | Path) -> str:
"""Extract the GNU BuildID from an ELF file."""
return self._get_build_id(path)
@property
def entries(self) -> list[DebugSymbolEntryType]:
"""The flattened list of parsed entries, each one describing a single ELF file."""
return self._entries
@property
def extra_input_files(self) -> set[Path]:
"""The set of extra input files read by the parsing functions.
These maybe be used to write implicit inputs in a Ninja depfile
for a script invoked through a GN action, to ensure proper
incremental builds.
"""
return self._visited_paths
def parse_manifest_file(self, manifest_path: Path) -> None:
"""Parse a given debug_symbols.json manifest.
Args:
manifest_path: Path to input manifest.
Raises:
ValueError if the manifest, or one of its includes is
malformed, or if there is a cycle in the include chain.
"""
if not manifest_path.exists():
raise ValueError(f"Missing manifest file at {manifest_path}")
with manifest_path.open("rt") as f:
manifest_json = json.load(f)
self.parse_manifest_json(manifest_json, manifest_path)
def _resolve_entry_build_id(self, entry: T.Any) -> str:
"""Return the build-id value of a specific manifest entry.
Args:
entry: A DebugSymbolEntryType value with a "debug" key.
Returns:
A build-id hexadecimal string on success, or an empty string
on failure, which can happen for non-ELF platforms, and a
certain number of entries like Go host binaries, or special
Zircon binaries.
"""
assert "debug" in entry, f"Invalid debug manifest entry {entry}"
# First, look at the debug value if it looks like ../.build-id/xx/yyyyyyyy.debug
debug = entry["debug"]
pos = debug.find(".build-id/")
if pos >= 0:
pos += len(".build-id/")
rest = debug[pos:]
if len(rest) > 3 and rest[2] == "/" and rest.endswith(".debug"):
build_id = rest[:2] + rest[3:-6]
return build_id
# Second, look for an elf_build_id_file path.
build_id_file = entry.get("elf_build_id_file")
if build_id_file:
build_id_file_path = self._build_dir / build_id_file
self._visited_paths.add(build_id_file_path)
# Some files list an elf_build_id_file but do not generate
# anything at that location. E.g.: XXXX
if build_id_file_path.exists():
build_id = build_id_file_path.read_text().strip()
# It is possible for certain files to have an empty
# build_id file, for example, see
# /zircon/kernel/lib/libc/string/arch/x86:_hermetic_code_blob.memcpy_movsb.executable(//zircon/kernel:kernel_x64)
if build_id:
return build_id
# As a fallback, try to extract the build id directly from the file.
debug_path = self._build_dir / debug
self._visited_paths.add(debug_path)
build_id = self._get_build_id(debug_path)
# It is possible for unstripped files to not have a GNU .build-id
# value, for example Go host binaries. So this could be an empty string.
return build_id
def _parse_entry(
self, entry: T.Any, manifest_path: Path | str | None
) -> None:
"""Parse a given debug manifest entry, and update state accordingly.
Args:
entry: The entry to parse
manifest_path: The path to use in error messages
"""
debug = entry.get("debug")
if debug:
if self._resolve_build_id and "elf_build_id" not in entry:
# Resolve build-id value, and skip entries for which this is
# not possible. This is expected and not worth logging since
# it covers hundreds of entries in a typical build, e.g. Go
# binaries and special Zircon binaries like hermetic code
# blobs.
build_id = self._resolve_entry_build_id(entry)
if not build_id:
return
entry["elf_build_id"] = build_id
self._entries.append(entry)
return
sub_manifest = entry.get("manifest")
if not sub_manifest:
raise ValueError(f"Malformed entry in {manifest_path}: {entry}")
sub_manifest_path = (self._build_dir / sub_manifest).resolve()
if not sub_manifest_path.exists():
raise ValueError(
f"Missing include path in {manifest_path}: {sub_manifest}"
)
# Check for cycles in manifest include chain. Raise exception if one
# is detected.
for n, visited in enumerate(self._visited_stack):
if visited == sub_manifest_path:
error_msg = "Recursive manifest includes:\n"
for visited in self._visited_stack[n:]:
error_msg += f" {visited} -->\n"
error_msg += f" {sub_manifest_path}"
raise ValueError(error_msg)
self._visited_paths.add(sub_manifest_path)
self._visited_stack.append(sub_manifest_path)
self.parse_manifest_file(sub_manifest_path)
self._visited_stack = self._visited_stack[:-1]
def parse_manifest_json(
self, manifest_json: T.Any, manifest_path: Path | str | None = None
) -> None:
"""Parse a given debug_symbols manifest JSON value.
Args:
manifest_json: The input manifest as a JSON value
(i.e. a list of dictionaries).
manifest_path: The path of the manifest whose content is
being parsed (only used for error messages).
Raises:
ValueError if the manifest, or one of its includes is
malformed, or if there is a cycle in the include chain.
"""
# As a special case, the prebuilt_binaries.json file can be 'None'
# instead of an empty list in a clean Fuchsia checkout that didn't
# use `jiri init -fetch-optional=debug-symbols`. Convert it here
# to avoid crashing the debug symbol extraction process on infra.
if manifest_json is None:
manifest_json = []
if not isinstance(manifest_json, list):
raise ValueError(
f"Malformed manifest at {manifest_path}: expected list, got {type(manifest_json)}"
)
for entry in manifest_json:
self._parse_entry(entry, manifest_path)
def deduplicate_entries(self) -> None:
"""Deduplicate entries after parsing.
Duplicate entries are possible when dealing with prebuilt symbols, for
example:
- The {BUILD}/gen/build/prebuilt/prebuilt_symbols.json file
is created by //build/prebuilt:prebuilt which parses the content
of //prebuilt/.build-id/ and generates breakpad symbol files
directly into {BUILD}/.build-id for them. The entries in this
manifest do not have a "label" key though. E.g.:
{
"breakpad": ".build-id/63/cd5f4d7cfa5a487cd50446b34023eb9a68fad6.sym",
"cpu": "arm64",
"debug": "../../prebuilt/.build-id/63/cd5f4d7cfa5a487cd50446b34023eb9a68fad6.debug",
"elf_build_id": "63cd5f4d7cfa5a487cd50446b34023eb9a68fad6",
"os": "fuchsia"
},
- The {BUILD}/obj/src/connectivity/bluetooth/core/bt-host/bt-host.debug_symbols.json
manifest contains labelled entries for some of the same prebuilt binaries,
but without any "breakpad" key.
{
"cpu": "arm64",
"debug": "../../prebuilt/.build-id/63/cd5f4d7cfa5a487cd50446b34023eb9a68fad6.debug",
"dest_path": "bin/bt_host",
"elf_build_id": "63cd5f4d7cfa5a487cd50446b34023eb9a68fad6",
"label": "//src/connectivity/bluetooth/core/bt-host:bt-host.package(//build/toolchain/fuchsia:arm64)",
"os": "fuchsia"
},
Deduplication verifies that there are no conflicts between entries pointing
to the same "debug" path, and merges the keys to keep all information.
Differing "label" values are not considered conflicts, and the first
one is kept.
This updates self._entries and should be called after a call to
parse_manifest_json() or parse_manifest_file().
Raises:
ValueError when duplicates are found. The exception string contains
a newline-separated list of error messages.
"""
debug_to_entries: dict[str, DebugSymbolEntryType] = {}
errors: list[str] = []
new_entries = []
for entry in self._entries:
debug = entry["debug"]
cur_entry = debug_to_entries.get(debug, None)
if cur_entry is None:
debug_to_entries[debug] = entry
new_entries.append(entry)
continue
for key in (
"elf_build_id",
"elf_build_id_file",
"breakpad",
"gsym",
"label",
"cpu",
"os",
"dest_path",
"stripped",
):
cur_value = cur_entry.get(key, "")
new_value = entry.get(key, "")
if cur_value == new_value:
continue
if cur_value == "":
cur_entry[key] = new_value
elif new_value != "" and key not in (
"label",
"dest_path",
"breakpad",
"gsym",
"stripped",
):
# A single binary can legitimately be referenced by
# multiple targets, e.g. a Bazel cc_binary used as the
# `binary` of several host_test() targets, or prebuilt
# packages with different destination paths. Differing
# labels, destination paths, breakpad, gsym, or stripped
# symbol paths are not a conflict; keep the first one.
errors.append(
f"Incompatible '{key}' value between {cur_entry} and {entry}"
)
# If the merge resolved the BuildID value, drop the elf_build_id_file key.
if cur_entry.get("elf_build_id") and cur_entry.get(
"elf_build_id_file"
):
cur_entry.pop("elf_build_id_file", None)
if errors:
raise ValueError("\n".join(errors))
self._entries = new_entries
# Type of a callable object used to start a new command. First parameter
# is a unique command id, second parameter is the cmd_data passed to
# CommandPool.add_command(). Must return a new subprocess.Popen instance.
CommandRunnerType = T.Callable[[int, T.Any], subprocess.Popen[str]]
# Type of a callable object used to compute the result of a given
# command invocation. First parameter is a unique command id, second
# parameter is a cmd_data value, and third parameter is a completed
# subprocess.Popen value.
CommandResultProcessorType = T.Callable[
[int, T.Any, subprocess.Popen[str]], T.Any
]
class CommandPool(object):
"""Generic parallel command runner class.
Usage is:
1) Create instance, passing the max parallel command count (depth).
2) Call add_command() as many time as necessary to schedule a
command to run. This takes a callable that must return
a subprocess.Popen instance to start the command.
3) Call run() to launch all commands in parallel. This generates
a sequence of (cmd_id, cmd_result) values. Where cmd_id is
a unique command index, and cmd_result is the result of calling
result_processor() on completed subprocess.Popen instances.
"""
def __init__(self, depth: int) -> None:
"""Create instance.
Args:
depth: Maximum number of parallel commands to run.
"""
self._depth = depth
self._commands: list[tuple[int, T.Any, CommandRunnerType]] = []
def add_command(
self, cmd_data: T.Any, cmd_runner: CommandRunnerType
) -> None:
"""Add a new command invocation to the scheduled plan.
Args:
runner: A callable that returns a new subprocess.Popen
instance obtained by calling subprocess.run() for
the command. It takes a unique command id as its
first parameter.
"""
cmd_id = len(self._commands)
self._commands.append((cmd_id, cmd_data, cmd_runner))
def run(
self, result_processor: CommandResultProcessorType
) -> T.Iterator[T.Any]:
"""Run all recorded commands in parallel.
This ensures that this will not launch more than self._depth
commands in parallel.
Args:
result_processor: A callable used to convert the result of a completed
command's Popen value into an result value.
Yields:
A sequence of cmd_result values, each one being computed by calling
result_processor() on command completion.
"""
# Map command ids to running subprocess.Popen instances.
running: dict[int, subprocess.Popen[str]] = {}
def poll_run_queue() -> T.Iterator[T.Any]:
# Find all completed commands.
results: list[T.Any] = []
completed_ids: list[int] = []
for cmd_id, proc in running.items():
returncode = proc.poll()
if returncode is not None:
cmd_data = self._commands[cmd_id][1]
completed_ids.append(cmd_id)
results.append(result_processor(cmd_id, cmd_data, proc))
# Release file descriptors as early as possible.
del proc
for cmd_id in completed_ids:
del running[cmd_id]
yield from results
if not completed_ids:
time.sleep(0.01) # 10ms
for cmd_id, cmd_data, cmd_runner in self._commands:
while len(running) == self._depth:
yield from poll_run_queue()
running[cmd_id] = cmd_runner(cmd_id, cmd_data)
while running:
yield from poll_run_queue()
class DebugSymbolExporter(object):
"""A class used to export debug symbols and generate breakpad symbols if needed.
Usage is:
1) Create instance. Optionally passing the path to the Breakpad `dump_syms` tool.
2) Call parse_debug_symbols(), passing a flat list of debug symbol
manifest entries whose "elf_build_id" key has been set properly
for ELF platforms.
3) Call export_debug_symbols() to populate a directory with symlinks
to debug symbol files (following the standard
$EXPORT_DIR/.build-id/xx/yyyyyyyyyyy.debug layout), as well as
$EXPORT_DIR/build-ids.json and $EXPORT_DIR/build-ids.txt.
If a `dump_syms` path was passed to the constructor, this also creates
Breakpad symbol files from the ELF debug one under the name
$EXPORT_DIR/.build-id/xx/yyyyyyyyyyyy.sym).
The directory's content is re-created on each call, and thus does not
accumulate files from previous runs.
"""
def __init__(
self,
build_dir: Path | str,
dump_syms_tool: None | Path | str = None,
gsymutil_tool: None | Path | str = None,
log: None | T.Callable[[str], None] = None,
log_error: None | T.Callable[[str], None] = None,
) -> None:
"""Create instance.
Args:
build_dir: The Ninja build directory path.
dump_syms_tool: Optional path the Breakpad dump_syms host tool.
If provided, will be used to generate breakpad symbol files
from ELF debug binaries.
gsymutil_tool: Optional path to the LLVM gsymutil tool host
tool. If provided, will be used to generate GSYM symbol files
from ELF debug binaries.
"""
self._build_dir = Path(build_dir)
# Map destination path, relative to self._output_dir, to target path.
# Used for debug symbol symlinks, as well as symlinks to existing breakpad symbol files.
self._symlink_map: dict[Path, Path] = {}
# Map breakpad symbol destination path, relative to self._output_dir, to
# the manifest entry.
self._breakpad_map: dict[Path, DebugSymbolEntryType] = {}
# A similar map for GSYM symbol extraction.
self._gsym_map: dict[Path, DebugSymbolEntryType] = {}
self._dump_syms_tool = dump_syms_tool
self._gsymutil_tool = gsymutil_tool
self._log = log
self._log_error = log_error
# Map GNU BuildId hex value to the corresponding GN or Bazel label.
# (Bazel labels always starts with @).
self._build_ids_to_labels: dict[str, str] = {}
# The set of BuildID values for Fuchsia binaries only.
self._fuchsia_build_ids: set[str] = set()
# Filtered list of debug symbol entries (only for ELF binaries containing
# a proper GNU BuildID note). "debug" paths will be relative to the output
# directory.
self._debug_symbols: list[DebugSymbolEntryType] = []
def parse_debug_symbols(
self, debug_entries: list[DebugSymbolEntryType]
) -> None:
"""Parse list of debug symbol entries.
Args:
debug_entries: A list of DebugSymbolEntryType values. Entries for non-ELF
systems will be ignored, otherwise their "elf_build_id" key must be
set properly.
"""
# Find all the debug symbol entries that do not have a breakpad symbol path yet.
for entry in debug_entries:
# Skip non-ELF platforms.
if entry["os"] not in ("fuchsia", "linux"):
continue
if os.path.basename(entry["debug"]) in _DEBUG_BINARY_DENYLIST:
continue
build_id = entry.get("elf_build_id", "")
if not build_id:
if self._log:
self._log(f"Missing elf_build_id in entry: {entry}")
continue
label = entry.get("label", "")
if build_id not in self._build_ids_to_labels or label:
self._build_ids_to_labels[build_id] = label
if entry["os"] == "fuchsia":
self._fuchsia_build_ids.add(build_id)
# Record a .debug symlink in the symlink map.
debug_dst_path = (
Path(".build-id") / build_id[:2] / f"{build_id[2:]}.debug"
)
debug_src_path = self._build_dir / entry["debug"]
self._symlink_map[debug_dst_path] = debug_src_path
# If a stripped file is provided, add a symlink to it as well.
if entry.get("stripped", "") != "":
stripped_src_path = self._build_dir / entry["stripped"]
stripped_dst_path = (
Path(".build-id") / build_id[:2] / f"{build_id[2:]}"
)
self._symlink_map[stripped_dst_path] = stripped_src_path
# If a breakpad file path is provided by the entry, record a .sym symlink in the
# map. Otherwise, add an entry in breakpad_map to ensure a breakpad symbol
# file can be created later. *Unless* this is not a Fuchsia binary, see
# https://fxbug.dev/445119122 for context.
breakpad_dst_path = Path(
str(debug_dst_path).replace(".debug", ".sym")
)
breakpad_file = entry.get("breakpad")
if breakpad_file:
self._symlink_map[breakpad_dst_path] = (
self._build_dir / breakpad_file
)
self._breakpad_map.pop(breakpad_dst_path, None)
elif (
entry["os"] == "fuchsia"
and self._dump_syms_tool
and breakpad_dst_path not in self._symlink_map
):
self._breakpad_map[breakpad_dst_path] = entry
# Same for GSYM symbols.
gsym_dst_path = Path(str(debug_dst_path).replace(".debug", ".gsym"))
gsym_file = entry.get("gsym")
if gsym_file:
self._symlink_map[gsym_dst_path] = self._build_dir / gsym_file
self._gsym_map.pop(gsym_dst_path, None)
elif (
entry["os"] == "fuchsia"
and self._gsymutil_tool
and gsym_dst_path not in self._symlink_map
):
self._gsym_map[gsym_dst_path] = entry
debug_symbol = entry.copy()
debug_symbol["debug"] = str(debug_dst_path)
# The following lines remove "stripped" if it is an empty string or
# otherwise rewrite it.
debug_symbol.pop("stripped", None)
if entry.get("stripped", "") != "":
debug_symbol["stripped"] = ".build-id/{}/{}".format(
build_id[:2], build_id[2:]
)
debug_symbol.pop("breakpad", None)
if (
breakpad_file
or breakpad_dst_path in self._symlink_map
or (entry["os"] == "fuchsia" and self._dump_syms_tool)
):
debug_symbol["breakpad"] = str(breakpad_dst_path)
debug_symbol.pop("elf_build_id_file", None)
self._debug_symbols.append(debug_symbol)
def get_debug_symbols_to_build_id_copies(
self, build_id_dir: Path | str
) -> list[tuple[str, str]]:
"""Get the list of files to copy to an external .build-id/ directory.
Collects the list of (src, dst) path pairs to copy debug binaries to their
location in an output .build-id directory. Useful if the caller wants to
perform the copy using parallel threads / tasks.
Consider using copy_debug_symbols_to_build_id() instead to do the copy.
Args:
build_id_dir: Path to the output .build-id directory. Its name must
be ".build-id".
Raises
AssertionError if build_id_dir.name is not ".build-id".
Returns:
A list of (src_path, dst_path) pairs describing which files to copy.
"""
build_id_dir = Path(build_id_dir)
assert (
build_id_dir.name == ".build-id"
), f"Invalid output directory name (.../.build-id expected): {build_id_dir}"
# dst_path values in the loop below already begin with .build-id/
build_id_root = build_id_dir.parent
result: list[tuple[str, str]] = []
for dst_path, target_path in self._symlink_map.items():
if not dst_path.name.endswith(".debug"):
continue # Ignore breakpad, GSYM or stripped files.
output_path = build_id_root / dst_path
if output_path.exists():
# The {BUILD_DIR}/.build-id/ directory accumulates files over time,
# unless the next `fx clean`.
#
# It is assumed that BuildID values are unique and content-dependent.
# Thus if a file already exists in the destination directory, assume
# this is a duplicate and ignore it.
continue
result.append((str(target_path), str(output_path)))
return result
def copy_debug_symbols_to_build_id(self, build_id_dir: Path | str) -> None:
"""Copy debug symbols to an external .build-id/ directory.
Unlike export_debug_symbols(), this does not create a new output directory,
and copies the file instead of creating symlinks.
This also doesn't create build-ids.txt, build-ids.json or breakpad files.
This should be used only to copy debug symbols to the Ninja build directory,
for local debugging / symbolification.
Args:
build_id_dir: Path to the output .build-id directory. Its name must be ".build-id".
"""
copies = self.get_debug_symbols_to_build_id_copies(build_id_dir)
log = self._log
for src, dst in copies:
src_path = Path(src)
dst_path = Path(dst)
if log:
log(f"Copying {dst_path} FROM {src_path}")
dst_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src_path, dst_path)
def verify_elf_sizes_debug_symbols(
self,
elf_sizes_paths: T.Sequence[Path] = (),
prebuilt_merkles: T.Container[str] = (),
exempt_build_ids: T.Container[str] = (),
prebuilt_build_id_dir: Path | None = None,
) -> bool:
"""Verify assembled target ELFs in elf_sizes.json have exported debug symbols.
Rules enforced:
1. Every source-built ELF in elf_sizes.json must be present in
exported debug symbols (self._build_ids_to_labels).
2. Every prebuilt ELF whose .debug file exists in prebuilt_build_id_dir
must be present in exported debug symbols.
3. Prebuilt ELFs without a .debug file in prebuilt_build_id_dir or
explicitly listed in exempt_build_ids (such as third-party vendor
binaries, OOT prebuilt package runtime libraries, local builds
without fetch-optional=debug-symbols, or prebuilt toolchain
multilib runtime libraries on sanitizer builds) are exempted.
Args:
elf_sizes_paths: Sequence of elf_sizes.json manifest paths to validate
against exported debug symbols.
prebuilt_merkles: Container of merkle roots belonging to prebuilt blobs.
exempt_build_ids: Container of GNU build-id hex strings exempted from
symbol verification (e.g. prebuilt toolchain runtime libraries).
prebuilt_build_id_dir: Optional path to //prebuilt/.build-id directory.
Returns:
True if all non-exempt assembled ELFs have exported debug symbols,
False otherwise.
"""
if not elf_sizes_paths:
return True
missing_errors: list[str] = []
for elf_sizes_path in elf_sizes_paths:
with elf_sizes_path.open("rt") as f:
elf_sizes_json = json.load(f)
for entry in elf_sizes_json.get("files", []):
build_id: str = entry.get("build_id", "")
if (
not build_id
or os.path.basename(entry.get("path", ""))
in _DEBUG_BINARY_DENYLIST
or build_id in self._build_ids_to_labels
or build_id in exempt_build_ids
):
continue
blob_path: str = entry.get("path", "<unknown>")
rel_debug = Path(build_id[:2]) / f"{build_id[2:]}.debug"
if os.path.basename(blob_path) in prebuilt_merkles:
if (
prebuilt_build_id_dir is not None
and (prebuilt_build_id_dir / rel_debug).exists()
):
missing_errors.append(
f" - build_id={build_id} path={blob_path} "
f"(prebuilt ELF has symbol file at prebuilt/.build-id/{rel_debug})"
)
continue
missing_errors.append(
f" - build_id={build_id} path={blob_path} (source-built ELF)"
)
if missing_errors:
if self._log_error:
self._log_error(
"Missing debug symbols in exported debug_symbols.json for assembled ELF binaries:\n"
+ "\n".join(missing_errors)
)
return False
return True
def export_debug_symbols(
self,
output_dir: Path | str,
depth: int = 0,
elf_sizes_paths: T.Sequence[Path] = (),
prebuilt_merkles: T.Container[str] = (),
exempt_build_ids: T.Container[str] = (),
prebuilt_build_id_dir: Path | None = None,
) -> bool:
"""Create a new output directory with symlinks to debug symbol files.
This populates output_dir to provide symlinks to debug symbols using
the standard .build-id/xx/yyyyyyyyyyy.debug layout for them.
If the input debug symbol entries contained paths to breakpad symbol files,
these will appear in the output directory as symlink named
.build-id/xx/yyyyyyyyyyy.sym
Finally, if dump_syms_tool was provided in the constructor, it will be
invoked to generate breakpad symbol files in the output directory, by
parsing the content of .build-id/xx/yyyyyyyyyy.debug to generate the
corresponding .build-id/xx/yyyyyyyyyyyy.sym file.
Args:
output_dir: Path to output directory. Its current content will be removed.
depth: Optional. Maximum number of concurrent dump_syms invocations to use.
Default is to use the number of available CPU cores.
elf_sizes_paths: Optional sequence of elf_sizes.json paths to validate against
exported debug symbols.
prebuilt_merkles: Optional container of merkle roots belonging to prebuilt blobs.
exempt_build_ids: Optional container of build IDs exempted from symbol verification
(such as prebuilt toolchain runtime shared libraries on sanitizer builds).
prebuilt_build_id_dir: Optional path to //prebuilt/.build-id directory.
Returns:
True on success, False on error.
On success, the output directory will contain:
- .build-id/xx/yyyyyyyyyyy.debug symlinks to the corresponding debug
symbol file.
- .build-id/xx/yyyyyyyyyyyy.sym breakpad symbol files. These are either
symlinks (when the breakpad symbol was provided by the Fuchsia checkout,
e.g. for Clang and Rust runtime libraries), or a file generated by this
function by invoking the Breakpad `dump_syms` tool, if provided.
- A build-ids.json file mapping hexadecimal build-id values to labels
of the generating target (either from GN or Bazel, Bazel labels always
start with @). This is used by one coverage-related infra recipe
- A build-ids.txt which lists all build-id values in the export directory,
one hexadecimal string per line. Used by artifactory to upload
the debug symbols to cloud storage.
- A debug_symbols.json manifest following the //:debug_symbols schema
detailing each debug binary, its build-id value, its optional Fuchsia
installation path, and its optional breakpad file.
"""
log = self._log
if not self.verify_elf_sizes_debug_symbols(
elf_sizes_paths=elf_sizes_paths,
prebuilt_merkles=prebuilt_merkles,
exempt_build_ids=exempt_build_ids,
prebuilt_build_id_dir=prebuilt_build_id_dir,
):
return False
output_dir = Path(output_dir)
# Create clean new output directory.
if output_dir.exists():
shutil.rmtree(output_dir)
output_dir.mkdir(parents=True)
# Write the build-ids.json file for artifactory.
build_ids_json = output_dir / "build-ids.json"
if log:
log(f"Creating {build_ids_json}")
with build_ids_json.open("wt") as f:
json.dump(self._build_ids_to_labels, f, sort_keys=True, indent=2)
# Write the build-ids.txt file for artifactory.
# Artifactory expects that this only contains values for Fuchsia binaries.
build_ids_txt = output_dir / "build-ids.txt"
if log:
log(f"Creating {build_ids_txt}")
with build_ids_txt.open("wt") as f:
for build_id in sorted(self._fuchsia_build_ids):
f.write(f"{build_id}\n")
if log:
log(
"Creating %d symlinks in %s"
% (len(self._symlink_map), output_dir)
)
# Populate all symlinks under output_dir
for dst_path, target_path in self._symlink_map.items():
link_path = output_dir / dst_path
link_path.parent.mkdir(parents=True, exist_ok=True)
link_path.symlink_to(target_path.resolve())
# Write the debug_symbols.json manifest.
with (output_dir / "debug_symbols.json").open("wt") as f:
json.dump(self._debug_symbols, f, indent=2, sort_keys=True)
if depth <= 0:
depth = len(os.sched_getaffinity(0))
if self._dump_syms_tool and not self._generate_breakpad_symbols(
output_dir, depth
):
return False
if self._gsymutil_tool and not self._generate_gsym_symbols(
output_dir, depth
):
return False
if log:
log("Done!")
return True
def _generate_breakpad_symbols(self, output_dir: Path, depth: int) -> bool:
"""Generate breakpad symbols from debug binaries.
Runs the dumpsym tool in multiple processes concurrently.
Args:
output_dir: Output directory path.
depth: Number of max concurrent processes allowed.
Returns:
True in case of success.
"""
log = self._log
log_error = self._log_error
if not self._breakpad_map:
if log:
log("Skipping breakpad symbol generation (no inputs).")
return True
if log:
log(
"Generating %d breakpad symbols in %s"
% (len(self._breakpad_map), output_dir)
)
command_pool = CommandPool(depth)
cmd_ids_to_files: dict[int, T.TextIO] = {}
def cmd_dumpsym_runner(
cmd_id: int, entry: DebugSymbolEntryType
) -> subprocess.Popen[str]:
"""Start a command to generate the breakpad symbol file for a given entry.
Args:
entry: The input debug symbol manifest entry.
Returns:
a new subprocess.Popen instance.
"""
debug_symbol_path = self._build_dir / entry["debug"]
build_id = entry["elf_build_id"]
symbol_path = (
Path(".build-id") / build_id[:2] / f"{build_id[2:]}.sym"
)
if log:
log(
f" - Creating {symbol_path} FROM {os.path.relpath(debug_symbol_path)}"
)
dumpsym_cmd = [
str(self._dump_syms_tool),
"-r", # Do not handle inter-compilation unit references.
]
if entry["os"] == "fuchsia":
dumpsym_cmd += [
"-n",
"<_>", # Use specific object name.
"-o",
"Fuchsia", # Use specific operating system name.
]
dumpsym_cmd += [str(debug_symbol_path)]
symbol_path = output_dir / symbol_path
symbol_path.parent.mkdir(parents=True, exist_ok=True)
symbol_path.unlink(missing_ok=True)
symbol_file = symbol_path.open("wt")
# Save for closing in the result processor.
cmd_ids_to_files[cmd_id] = symbol_file
return subprocess.Popen(
dumpsym_cmd,
stdout=symbol_file,
stderr=subprocess.PIPE,
text=True,
)
def cmd_dumpsym_result_processor(
cmd_id: int,
entry: DebugSymbolEntryType,
proc: subprocess.Popen[str],
) -> bool:
"""Process the result of a completed command."""
assert proc.returncode is not None
assert proc.stderr # make pypy happy.
stderr = proc.stderr.read()
proc.stderr.close()
# Close the output file handle.
cmd_ids_to_files[cmd_id].close()
if proc.returncode == 0:
return True
if log_error:
log_error(
"Failure to generate breakpad symbol for: {}".format(
entry["debug"]
)
)
log_error(stderr)
return False
for entry in self._breakpad_map.values():
command_pool.add_command(entry, cmd_dumpsym_runner)
# Run all commands in parallel and change result to False if at
# least one error is detected. Error messages are sent to log_error
# by cmd_result_processor() directly.
success = True
for cmd_success in command_pool.run(cmd_dumpsym_result_processor):
if not cmd_success:
success = False
return success
def _generate_gsym_symbols(self, output_dir: Path, depth: int) -> bool:
"""Generate breakpad symbols from debug binaries.
Runs the gsymutil tool in multiple processes concurrently.
Args:
output_dir: Output directory path.
depth: Number of max concurrent processes allowed.
Returns:
True in case of success.
"""
log = self._log
log_error = self._log_error
if not self._gsym_map:
if log:
log("Skipping GSYM symbol generation (no inputs).")
return True
if log:
log(
"Generating %d GSYM symbols in %s"
% (len(self._gsym_map), output_dir)
)
command_pool = CommandPool(depth)
def cmd_gsymutil_runner(
cmd_id: int, entry: DebugSymbolEntryType
) -> subprocess.Popen[str]:
"""Start a command to generate the GSYM symbol file for a given entry.
Args:
entry: The input debug symbol manifest entry.
Returns:
a new subprocess.Popen instance.
"""
src_symbol_path = self._build_dir / entry["debug"]
build_id = entry["elf_build_id"]
gsymbol_path = ".build-id/{}/{}.gsym".format(
build_id[:2], build_id[2:]
)
if log:
log(
f" - Creating {gsymbol_path} FROM {os.path.relpath(src_symbol_path)}"
)
dst_symbol_path = output_dir / gsymbol_path
gsymutil_cmd = [
str(self._gsymutil_tool),
f"--convert={src_symbol_path}",
f"--out-file={dst_symbol_path}",
]
dst_symbol_path.parent.mkdir(parents=True, exist_ok=True)
dst_symbol_path.unlink(missing_ok=True)
return subprocess.Popen(
gsymutil_cmd,
# Do not capture stdout with subprocess.PIPE here as it is often
# larger than the pipe's capacity (64 kiB on Linux), blocking the
# process and preventing it from ever completing.
# The alternative would be to redirect to an opened file object
# but there is no need for this output anyway.
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
)
def cmd_gsymutil_result_processor(
cmd_id: int,
entry: DebugSymbolEntryType,
proc: subprocess.Popen[str],
) -> bool:
"""Process the result of a completed command."""
assert proc.returncode is not None
assert proc.stderr # make pypy happy.
stderr = proc.stderr.read()
proc.stderr.close()
if proc.returncode == 0:
return True
if log_error:
log_error(stderr)
return False
for entry in self._gsym_map.values():
command_pool.add_command(entry, cmd_gsymutil_runner)
# Run all commands in parallel and change result to False if at
# least one error is detected. Error messages are sent to log_error
# by cmd_result_processor() directly.
success = True
for cmd_success in command_pool.run(cmd_gsymutil_result_processor):
if not cmd_success:
success = False
return success