blob: 7331d6d277b5b5828cfab0d9614febc7950e402f [file]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Verify the content of ELF binaries embedded in a package or filesystem.
This scripts takes as input a FINI manifest file and on success, will write
an empty stamp file.
"""
# System imports
import argparse
import os
import sys
from collections.abc import Iterable
from typing import Any
from .elfinfo import get_elf_info
# The general strategy for checking the ELF binaries within a package is
# the following:
#
# 1) libzircon.so should never appear inside a Fuchsia package, since
# it is injected by the kernel into user processes at runtime. Its
# presence in a manifest is a build error.
#
# On the other hand, it will be listed as a dependency for most
# binaries, but not necessarily all of them, and should be ignored
# when that happens.
#
# 2) ld.so.1 is the dynamic loader and also serves as the C library,
# which means any DT_NEEDED reference to libc.so is automatically
# resolved at runtime to ld.so.1 instead.
#
# It is thus a build error to have libc.so inside a Fuchsia package,
# because the dynamic loader will never load these files!
#
# 3) ELF executables have a PT_INTERP section that will contain 'ld.so.1'
# for Fuchsia regular binaries, and '<variant>/ld.so.1' for those built
# with an instrumented variant (e.g. 'asan'). The library loader will
# search for DT_NEEDED dependencies in '/lib' in the first case,
# and '/lib/<variant>' in the second case, so check that these are
# properly located there, as well as their own dependencies.
#
# For now, it is an error to have a different PT_INTERP value, but
# this may be relaxed in the future, for example to package Linux
# and Android executables to be run inside Fuchsia containers.
#
# 4) Loadable modules are ELF shared libraries that are loaded at runtime
# by binaries (or the Vulkan loader service) through dlopen(), these
# can be located anywhere in the package. They may have DT_NEEDED
# dependencies which will be looked up at runtime in the loading
# executable's own library directory (i.e. '/lib' or '/lib/<variant>').
#
# Because our build system is flexible enough to group regular and
# instrumented executables in the same package, there is no 100%
# fool-proof way to know which executable will load the module.
#
# Moreover, one package can even contain a standalone module, without
# its own dependencies, to be loaded from another package's component
# which would provide them at runtime. This can happen for driver
# modules for example.
#
# Due to this, there is no attempt here to verify the dependencies
# of modules, since doing so can only be performed with heuristics
# that will fail at some point
#
# 5) Ignore files under data/, meta/ and lib/firmware, as they could be
# any ELF binaries for non-Fuchsia systems, or for testing.
#
# 6) Verify that unstripped files for binaries are provided
# in the build root directory, or toolchain-specific lib directories.
# Prebuilts are an exception, and are allowed not to have unstripped files.
# When .build-id directories are given (the Bazel build), the debug file
# is instead looked up by build-id in those directories.
#
def rewrite_elf_needed(dep: str) -> str | None:
"""Rewrite a DT_NEEDED dependency name.
Args:
dep: dependency name as it appears in ELF DT_NEEDED entry (e.g. 'libc.so')
Returns:
None if the dependency should be ignored, or the input dependency name,
possibly rewritten for specific cases (e.g. 'libc.so' -> 'ld.so.1')
"""
if dep == "libzircon.so":
# libzircon.so being injected by the kernel into user processes, it should
# not appear in Fuchsia packages, and thus should be ignored.
return None
if dep == "libc.so":
# ld.so.1 acts as both the dynamic loader and C library, so any reference
# to libc.so should be rewritten as 'ld.so.1'
return "ld.so.1"
# For all other cases, just return the unmodified dependency name.
return dep
def verify_elf_dependencies(
binary: str, lib_dir: str, deps: Iterable[str], elf_entries: dict[str, Any]
) -> tuple[list[str], list[str]]:
"""Verify the ELF dependencies of a given ELF binary.
Args:
binary: Name of the binary whose dependencies are verified.
lib_dir: The target directory where all dependencies should be.
deps: An iteration of dependency names, as they appear in DT_NEEDED
entries.
elf_entries: The global {target_path: elf_info} map.
Returns:
An (errors, libraries) tuple, where 'errors' is a list of error messages
in case of failure, or an empty list on succcess, and 'libraries' is a
list of libraries that were checked by this function.
"""
# Note that we do allow circular dependencies because they do happen
# in practice. In particular when generating instrumented binaries,
# e.g. for the 'asan' case (omitting libzircon.so):
#
# libc.so (a.k.a. ld.so.1)
# ^ ^ ^ |
# | | | v
# | | libclang_rt.asan.so
# | | | ^ ^
# | | v | |
# | libc++abi.so |
# | | |
# | v |
# libunwind.so-----------'
#
errors = []
libraries = []
visited = set()
queue = list(deps)
while len(queue) > 0:
dep = queue[0]
queue = queue[1:]
dep2 = rewrite_elf_needed(dep)
if dep2 is None or dep2 in visited:
continue
visited.add(dep2)
dep_target = os.path.join(lib_dir, dep2)
info = elf_entries.get(dep_target)
if not info:
errors.append("%s missing dependency %s" % (binary, dep_target))
else:
libraries.append(dep_target)
for subdep in info.needed:
if subdep not in visited:
queue.append(subdep)
return (errors, libraries)
def find_unstripped_file(
filename: str, depfile_items: set[str], toolchain_lib_dirs: list[str] = []
) -> str | None:
"""Find the unstripped version of a given ELF binary.
Args:
filename: input file path in build directory.
toolchain_lib_dirs: a list of toolchain-specific lib directories
which will be used to look for debug/.build-id/xx/xxxxxxx.debug
files corresponding to the input files's build-id value.
depfile_items: the set of examined files to be updated if needed.
Returns
Path to the debug file, if it exists, or None.
"""
if os.path.exists(filename + ".debug"):
# Zircon-specific toolchains currently write the unstripped files
# for .../foo as .../foo.debug.
debugfile = filename + ".debug"
else:
# Check for toolchain runtime libraries, which are stored under
# {toolchain}/lib/.../libfoo.so, and whose unstripped file will
# be under {toolchain}/lib/debug/.build-id/xx/xxxxxx.debug.
lib_dir = None
for dir in toolchain_lib_dirs:
if os.path.realpath(filename).startswith(
os.path.realpath(dir) + os.sep
):
lib_dir = dir
break
if lib_dir:
build_id_dir = os.path.join(lib_dir, "debug", ".build-id")
if not os.path.exists(build_id_dir):
# Local rust builds don't contain debug in the path, so fallback
# to a path without debug.
build_id_dir = os.path.join(lib_dir, ".build-id")
if not os.path.exists(build_id_dir):
return None
info = get_elf_info(filename)
assert info is not None, "%s: not an ELF file" % filename
build_id = info.build_id
# The build-id value is an hexadecimal string, used to locate the
# debug file under debug/.build-id/XX/YYYYYY.debug where XX are its
# first two chars, and YYYYYY is the rest (typically longer than
# 6 chars).
debugfile = os.path.join(
build_id_dir, build_id[:2], build_id[2:] + ".debug"
)
if not os.path.exists(debugfile):
return None
# Pass filename as fallback so we don't fallback to the build-id entry name.
return debugfile
else:
# Otherwise, the Fuchsia build places unstripped files under
# .../lib.unstripped/foo.so (for shared library or loadable module
# .../foo.so) or .../exe.unstripped/bar (for executable .../bar).
dir, file = os.path.split(filename)
if file.endswith(".so") or ".so." in file:
subdir = "lib.unstripped"
else:
subdir = "exe.unstripped"
debugfile = os.path.join(dir, subdir, file)
if (
subdir == "lib.unstripped"
and not os.path.exists(debugfile)
and os.path.basename(dir) == "dist"
):
# Rust dylibs emit a stripped distribution copy under
# .../dist/libfoo.so while keeping the unstripped file under
# .../lib.unstripped/libfoo.so.
dist_debugfile = os.path.join(
os.path.dirname(dir), subdir, file
)
if os.path.exists(dist_debugfile):
debugfile = dist_debugfile
while not os.path.exists(debugfile):
# For dir/foo/bar, if dir/foo/exe.unstripped/bar
# didn't exist, try dir/exe.unstripped/foo/bar.
parent, dir = os.path.split(dir)
if not parent or not dir:
return None
dir, file = parent, os.path.join(dir, file)
debugfile = os.path.join(dir, subdir, file)
if not os.path.exists(debugfile):
debugfile = os.path.join(subdir, filename)
if not os.path.exists(debugfile):
return None
depfile_items.add(debugfile)
return debugfile
class VerificationFailure(Exception):
def __init__(
self,
errors: list[str],
extras: list[str] | None = None,
) -> None:
self.errors = errors
self.extras = extras
def __str__(self) -> str:
lines = self.errors.copy()
if self.extras:
lines.extend(self.extras)
return "\n".join(lines)
def find_debug_file_by_build_id(
info: Any,
build_id_dirs: list[str],
debug_files_by_build_id: dict[str, str],
depfile_items: set[str],
) -> str | None:
"""Find the debug file of an ELF binary by build-id.
Args:
info: The ElfInfo value of the binary.
build_id_dirs: A list of .build-id directories, each of which holds
debug files at xx/yyyyyy.debug, where xx are the first two hex chars
of the build-id and yyyyyy is the rest.
debug_files_by_build_id: A {build_id: path} map of other debug files.
depfile_items: the set of examined files to be updated if needed.
Returns:
None if the debug file was found, or an error message otherwise.
"""
build_id = info.build_id
if not build_id:
return "%s has no build-id, so its debug file cannot be found" % (
info.filename
)
debugfile = debug_files_by_build_id.get(build_id)
if debugfile:
depfile_items.add(debugfile)
return None
debug_path = os.path.join(build_id[:2], build_id[2:] + ".debug")
for build_id_dir in build_id_dirs:
debugfile = os.path.join(build_id_dir, debug_path)
if os.path.exists(debugfile):
depfile_items.add(debugfile)
return None
return "No debug file with build-id %s found for %s in: %s" % (
build_id,
info.filename,
", ".join(build_id_dirs + sorted(debug_files_by_build_id.values())),
)
def verify_manifest_elf_binaries(
manifest_entries: dict[str, str],
toolchain_lib_dirs: list[str],
build_id_dirs: list[str] | None = None,
debug_files: list[str] | None = None,
) -> set[str]:
"""Verify a set of manifest entries, throwing a VerificationFailure exception on failure.
Args:
manifest_entries: A {target_path: source_path} map of package entries.
toolchain_lib_dirs: Toolchain-provided lib directories used to locate
the unstripped files of toolchain runtime libraries.
build_id_dirs: If not None, locate the debug file of each ELF binary by
its build-id in these .build-id directories, instead of guessing the
path of its unstripped file. Bazel uses this, because its output
layout doesn't match the paths that find_unstripped_file() expects.
debug_files: Additional unstripped files that are looked up by build-id
along with build_id_dirs, for prebuilt stripped binaries whose
unstripped files are not in a .build-id directory (e.g. ld.so.1).
Returns a set of inputs for depfile inclusion on verification success.
"""
depfile_items = set()
debug_files_by_build_id: dict[str, str] = {}
for debug_file in debug_files or []:
debug_info = get_elf_info(debug_file)
if debug_info is not None and debug_info.build_id:
debug_files_by_build_id[debug_info.build_id] = debug_file
# Filter the input manifest entries to keep only the ELF ones
# that are not under data/, lib/firmware/ or meta/
elf_entries = {}
for target, source_file in manifest_entries.items():
if (
target.startswith("data/")
or target.startswith("lib/firmware/")
or target.startswith("meta/")
):
continue
depfile_items.add(source_file)
info = get_elf_info(source_file)
if info is not None:
elf_entries[target] = info
# errors contains a list of error strings corresponding to issues found in
# the input manifest.
#
# extras contains non-error strings that are useful to dump when an error
# occurs, and give more information about what was found in the manifest.
# These should only be printed in case of errors, or ignored otherwise.
errors = []
extras = []
# The set of all libraries visited when checking dependencies.
visited_libraries = set()
# Verify that libzircon.so or libc.so do not appear inside the package.
for target, info in elf_entries.items():
filename = os.path.basename(target)
if filename in ("libzircon.so", "libc.so"):
errors.append(
"%s should not be in this package (source=%s)!"
% (target, info.filename)
)
# First verify all executables, since we can find their library directory
# from their PT_INTERP value, then check their dependencies recursively.
for target, info in elf_entries.items():
interp = info.interp
if interp is None:
continue
interp_name = os.path.basename(interp)
if interp_name != "ld.so.1":
errors.append(
"%s has invalid or unsupported PT_INTERP value: %s"
% (target, interp)
)
continue
lib_dir = os.path.join("lib", os.path.dirname(interp))
extras.append(
"Binary %s has interp %s, lib_dir %s" % (target, interp, lib_dir)
)
binary_errors, binary_deps = verify_elf_dependencies(
target, lib_dir, info.needed, elf_entries
)
errors += binary_errors
visited_libraries.update(binary_deps)
# Check that all binaries are stripped
for target, info in elf_entries.items():
if not info.stripped:
errors.append("%s is not stripped: %s" % (target, info.filename))
# Verify that all ELF files have their unstripped file available.
for target, source_file in manifest_entries.items():
# Prebuilts are allowed to have missing debug files.
# See: https://fxbug.dev/42170486
if "/prebuilt/" in source_file:
continue
if target not in elf_entries:
continue
if build_id_dirs is not None:
# Bazel source paths are relative to the execroot (so they lack
# the `../../` prefix that makes GN paths contain "/prebuilt/"),
# and Bazel reaches prebuilts through external repositories that
# are symlinks into //prebuilt.
if "/prebuilt/" in "/" + source_file or "/prebuilt/" in (
os.path.realpath(source_file)
):
continue
error = find_debug_file_by_build_id(
elf_entries[target],
build_id_dirs,
debug_files_by_build_id,
depfile_items,
)
if error:
errors.append(error)
continue
unstripped = find_unstripped_file(
source_file, depfile_items, toolchain_lib_dirs
)
if unstripped is None:
errors.append("No unstripped file found for " + source_file)
if errors:
raise (VerificationFailure(errors, extras))
return depfile_items
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--source-dir",
default=".",
help="Root directory for source paths. Default is current directory.",
)
parser.add_argument(
"--toolchain-lib-dir",
default=[],
action="append",
metavar="DIR",
help="Path to toolchain-provided lib directory. Can be used multiple times.",
)
group = parser.add_mutually_exclusive_group()
group.add_argument("--fini-manifest", help="Input FINI manifest.")
group.add_argument(
"--partial-manifest", help="Input partial distribution manifest."
)
parser.add_argument("--stamp", required=True, help="Output stamp file.")
parser.add_argument("--depfile", help="Output Ninja depfile.")
args = parser.parse_args()
# Read the input manifest into a {target: source} dictionary.
manifest_entries: dict[str, str] = {}
input_manifest = args.fini_manifest
with open(input_manifest) as f:
for line in f:
line = line.rstrip()
target, _, source = line.partition("=")
assert source is not None, "Invalid manifest line: [%s]" % line
source_file = os.path.join(args.source_dir, source)
# Duplicate entries happen in some manifests, but they will have the
# same content, so assert otherwise.
if target in manifest_entries:
assert (
manifest_entries[target] == source_file
), 'Duplicate entries for target "%s": %s vs %s' % (
target,
source_file,
manifest_entries[target],
)
assert os.path.exists(source_file), (
"Missing source file for manifest line: %s" % line
)
manifest_entries[target] = source_file
try:
inputs = verify_manifest_elf_binaries(
manifest_entries, args.toolchain_lib_dir
)
except VerificationFailure as e:
print(e, file=sys.stderr)
return 1
if args.depfile:
with open(args.depfile, "w") as f:
f.write("%s: %s\n" % (input_manifest, " ".join(sorted(inputs))))
# Write the stamp file on success.
with open(args.stamp, "w") as f:
f.write("")
return 0
if __name__ == "__main__":
sys.exit(main())