build/bazel/scripts/bazel_tool.py

#!/usr/bin/env fuchsia-vendored-python
# Copyright 2023 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# IMPORTANT: This script should not depend on any Bazel workspace setup
# or specific environment. Only depend on standard Python3 modules
# and assume a minimum of Python3.8 is being used.

"""A tool to perform useful Bazel operations easily."""

import argparse
import os
import re
import subprocess
import sys
from pathlib import Path

_SCRIPT_DIR = os.path.dirname(__file__)
_BUILD_API_DIR = os.path.join(_SCRIPT_DIR, "../../api")
sys.path.insert(0, _BUILD_API_DIR)
from script_commands import ScriptCommandBase, ScriptCommandList


def error_message(msg: str) -> int:
    """Print error message to stderr, then return 1."""
    print(f"ERROR: {msg}", file=sys.stderr)
    return 1


def make_bazel_quiet_command(bazel: str, command: str) -> list[str]:
    """Create command argument list for a Bazel command that does not print too much.

    Args:
        bazel: Path to Bazel program.
        command: Bazel command (e.g. 'query', 'build', etc..)
    Returns:
        A sequence of strings that can be used as a command line prefix.
    """
    result = [
        bazel,
        command,
        "--noshow_loading_progress",
        "--noshow_progress",
        "--ui_event_filters=-info",
    ]
    if command != "query":
        result += ["--show_result=0"]
    return result


class TargetDumpCommand(ScriptCommandBase):
    """Dump definitions of Bazel targets."""

    DESCRIPTION_RAW = """
Print the real definition of a target, or set of targets, in a
Bazel graph, after all macros have been expanded. Note that:

- Each TARGET_SET argument is a standard Bazel target set expression
  (e.g. `//src/lib:foo` or `//:*`).

- The output is pretty printed for readability.

- For each target that is the result of a macro expansion, comments
  are added to describe the call chain that led to it.

- For each target generated by native.filegroup() in macros, new
  `generator_{function,location,name}` fields are added to indicate
  how it was generated.
"""

    @staticmethod
    def add_arguments(parser: argparse.ArgumentParser) -> None:
        parser.add_argument(
            "target_set",
            metavar="TARGET_SET",
            nargs="+",
            help="Set of targets to dump.",
        )

    @staticmethod
    def run(args: argparse.Namespace) -> int:
        bazel_cmd = (
            make_bazel_quiet_command(args.bazel, "query")
            + ["--output=build"]
            + args.target_set
        )
        buildifier_cmd = [
            args.buildifier,
            "--type=build",
            "--mode=fix",
            "--lint=off",
        ]
        proc1 = subprocess.Popen(
            bazel_cmd,
            stdin=subprocess.DEVNULL,
            stdout=subprocess.PIPE,
            stderr=None,
        )
        proc2 = subprocess.Popen(
            buildifier_cmd, stdin=proc1.stdout, stdout=None, stderr=None
        )
        proc1.wait()
        proc2.wait()
        if proc1.returncode != 0:
            return proc1.returncode
        return proc2.returncode


class ActionsCommand(ScriptCommandBase):
    """Dump action commands of Bazel targets."""

    DESCRIPTION_RAW = """
Print the action commands of a target, or set of targets, omitting
from the list of inputs sysroot and toolchain related headers, which
can be _very_ long when using the Fuchsia C++ toolchain.
"""

    @staticmethod
    def add_arguments(parser: argparse.ArgumentParser) -> None:
        parser.add_argument(
            "target_set",
            metavar="TARGET_SET",
            nargs="+",
            help="Set to targets to print actions for.",
        )
        parser.add_argument(
            "--",
            dest="extra_args",
            default=[],
            nargs=argparse.REMAINDER,
            help="extra Bazel aquery-compatible arguments.",
        )

    @staticmethod
    def run(args: argparse.Namespace) -> int:
        bazel_cmd = (
            make_bazel_quiet_command(args.bazel, "aquery")
            + args.extra_args[1:]
            + args.target_set
        )
        print("CMD %s" % bazel_cmd)
        ret = subprocess.run(
            bazel_cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True
        )
        if ret.returncode != 0:
            print("%s\n" % ret.stdout, file=sys.stdout)
            print("%s\n" % ret.stderr, file=sys.stderr)
            return ret.returncode

        def ignored_path(path: str) -> bool:
            if path.startswith(
                (
                    "external/prebuilt_clang/",
                    "external/fuchsia_clang/",
                    "prebuilt/third_party/sysroot/",
                )
            ):
                return True

            if path.find("external/fuchsia_prebuilt_rust/") >= 0:
                return True

            return False

        inputs_re = re.compile(r"^  Inputs: \[(.*)\]$")
        for line in ret.stdout.splitlines():
            m = inputs_re.match(line)
            if m:
                input_paths = []
                for path in m.group(1).split(","):
                    path = path.strip()
                    if not ignored_path(path):
                        input_paths.append(path)

                line = "  Inputs: [%s]" % (", ".join(input_paths))
            print(line)
        return 0


class SetGnTargetsCommand(ScriptCommandBase):
    """Update the @gn_targets symlink for a given Bazel target."""

    DESCRIPTION_RAW = """
Ensure the @gn_targets repository symlink points to the content required to
build a given Bazel target.

  fx bazel-tool set_gn_targets //vendor/acme/products/coyote:product_bundle

This is required before calling `fx bazel build <bazel_target>` directly
to ensure any filegroup in the @gn_targets repository points to the correct
up-to-date artifacts in the Ninja build directory.
"""

    @staticmethod
    def add_arguments(parser: argparse.ArgumentParser) -> None:
        parser.add_argument(
            "bazel_target",
            metavar="BAZEL_TARGET",
            help="A Bazel target label, must begin with // or @",
        )

    @staticmethod
    def run(args: argparse.Namespace) -> int:
        # Lazy import of modules.
        sys.path.insert(0, _SCRIPT_DIR)
        import bazel_action_utils
        import build_utils

        verbosity = args.verbose - args.quiet

        def log(level: int, msg: str) -> None:
            if verbosity >= level:
                print(msg)

        # Determine Ninja build directory
        if args.build_dir:
            build_dir = Path(args.build_dir).resolve()
        else:
            build_dir = build_utils.find_fx_build_dir(args.fuchsia_dir)
            if not build_dir:
                return error_message(
                    "Could not find Fuchsia build directory, use --build-dir=BUILD_DIR."
                )

            log(2, f"Found build directory: {build_dir}")

        bazel_target = args.bazel_target
        if not bazel_target.startswith(("//", "@")):
            return error_message(f"Invalid Bazel target label: {bazel_target}")
        if "(" in bazel_target:
            return error_message(
                f"Target label cannot use a GN toolchain suffix: {bazel_target}"
            )
        if bazel_target[0] == "@//":
            bazel_target = bazel_target[1:]

        def format_targets_list(targets: list[str]) -> str:
            """Helper to pretty-print a list of GN or Bazel targets."""
            return "\n".join(f"  {target}" for target in targets)

        # Load actions map then use it.
        actions_map = (
            bazel_action_utils.BazelBuildActionsMap.create_from_build_dir(
                build_dir
            )
        )

        bazel_paths = build_utils.BazelPaths(args.fuchsia_dir, build_dir)

        if args.bazel:
            bazel_launcher = args.bazel
        else:
            bazel_launcher = bazel_paths.launcher
            if not bazel_launcher.exists():
                return error_message(
                    "Cannot find Bazel launcher, use --bazel=PATH"
                )

        errors: list[str] = []

        gn_actions = bazel_action_utils.find_gn_bazel_action_infos_for(
            args.bazel_target,
            actions_map,
            build_utils.BazelLauncher(bazel_launcher),
            log=lambda msg: log(2, msg),
            log_err=lambda msg: errors.append(msg),
        )

        if errors:
            print(f"ERROR: {errors[0]}", file=sys.stderr)
            for error in errors[1:]:
                print(error, file=sys.stderr)
            return 1

        if not gn_actions:
            return error_message(
                f"This Bazel target is not a dependency of any known GN bazel_action() target: {bazel_target}\n"
                + "\nIt should be a dependency of one of the following Bazel targets:\n"
                + format_targets_list(actions_map.bazel_targets)
                + "\n\nWhich are built by one of these GN targets:\n"
                + format_targets_list(actions_map.gn_targets)
                + "\n"
            )

        if len(gn_actions) > 1:
            top_level_bazel_targets = []
            gn_targets = []
            for gn_action in gn_actions:
                top_level_bazel_targets += gn_action.bazel_targets
                gn_targets.append(gn_action.gn_target)

            return error_message(
                f"Several GN targets depend on {bazel_target}\n"
                + "\nChoose a higher Bazel target, one of:\n"
                + format_targets_list(sorted(top_level_bazel_targets))
                + "\n\nWhich are built by one of these GN targets:\n"
                + format_targets_list(sorted(gn_targets))
                + "\n"
            )

        gn_action = gn_actions[0]
        if (
            verbosity >= 1
        ):  # Do not generate large strings if they are not needed.
            log(
                1,
                f"Bazel target {bazel_target} is a dependency of GN target {gn_action.gn_target}\nWhich builds the following bazel targets:\n%s\n"
                % format_targets_list(gn_action.bazel_targets),
            )

        gn_targets_dest = actions_map.update_gn_targets_symlink(
            gn_action.gn_target, bazel_paths
        )
        assert gn_targets_dest, f"Could update create @gn_targets symlink"

        log(0, f"@gn_targets now points to {gn_targets_dest}")
        return 0


class ListBazelHostTestsCommand(ScriptCommandBase):
    """Generate tests.json file listing all Bazel host tests."""

    @staticmethod
    def run(args: argparse.Namespace) -> int:
        import json

        import bazel_tests_utils
        import build_utils

        bazel_paths = build_utils.BazelPaths(args.fuchsia_dir, args.build_dir)
        tests_json, _ = bazel_tests_utils.generate_tests_json(bazel_paths)

        print(json.dumps(tests_json, indent=2))
        return 0


class ExpandCommand(ScriptCommandBase):
    """Expand Bazel action's command lines."""

    DESCRIPTION_RAW = """
This command runs `bazel aquery` on a given target to extract its compilation or
linking command lines, while expanding in the output the content of response files
that are used to implement the build_flags() feature.

In the presence of response files, this will try to read them directly from the
Bazel execroot. This requires the target having been built to ensure their
content is correct.

Example usage:

    fx bazel-tool expand //build/bazel/host_tests -- --config=host
"""

    @staticmethod
    def _format_human_command(
        target: str,
        config: str,
        mnemonic: str,
        args: list[str],
        env_vars: list[str] | None = None,
        warnings: list[str] | None = None,
    ) -> str:
        import shlex

        quoted_envs = [shlex.quote(env) for env in env_vars] if env_vars else []
        quoted_args = [shlex.quote(arg) for arg in args]

        full_command_line = []
        if quoted_envs:
            full_command_line = ["env"] + quoted_envs
        full_command_line.extend(quoted_args)

        out = []
        out.append("=========================================")
        out.append(f"Mnemonic: {mnemonic}")
        out.append(f"Target: {target}")
        out.append(f"Configuration: {config}")
        if warnings:
            for warning in warnings:
                out.append(f"WARNING: {warning}")
        out.append("=========================================")

        if full_command_line:
            formatted_cmd = [full_command_line[0]]
            for arg in full_command_line[1:]:
                formatted_cmd[-1] += " \\"
                formatted_cmd.append(f"  {arg}")
            out.extend(formatted_cmd)

        out.append("=========================================\n")
        return "\n".join(out)

    @staticmethod
    def add_arguments(parser: argparse.ArgumentParser) -> None:
        parser.add_argument(
            "--format",
            choices=["human", "json"],
            default="human",
            help="Output format (human-readable or JSON)",
        )
        parser.add_argument(
            "--mnemonic",
            help="Filter actions to only those matching the given mnemonic name(s) (comma-separated, e.g., CppCompile,CppLink,Rustc)",
        )
        parser.add_argument(
            "target_label", help="Bazel target label to inspect."
        )
        parser.add_argument(
            "--",
            dest="extra_args",
            default=[],
            nargs=argparse.REMAINDER,
            help="extra Bazel query-compatible arguments (e.g. --config=host).",
        )

    @staticmethod
    def run(args: argparse.Namespace) -> int:
        import json

        import bazel_build_args
        import build_utils

        target = args.target_label
        if not target.startswith(("//", "@")):
            return error_message(f"Invalid Bazel target label: {target}")
        if "(" in target:
            return error_message(
                f"Target label cannot use a GN toolchain suffix: {target}"
            )

        # Auto-detect build environment and paths
        try:
            bazel_paths = build_utils.BazelPaths.new(
                fuchsia_dir=args.fuchsia_dir, build_dir=args.build_dir
            )
        except ValueError as e:
            return error_message(str(e))

        execroot = str(bazel_paths.execroot)
        bazel_launcher = build_utils.BazelLauncher(bazel_paths.launcher)

        # Extract extra config args from -- if passed
        config_args = []
        if args.extra_args:
            # The first element of extra_args is the "--" separator.
            config_args = args.extra_args[1:]

        filter_mnemonics = None
        if args.mnemonic:
            filter_mnemonics = [m.strip() for m in args.mnemonic.split(",")]

        # Perform the aquery and expand its results.
        expanded_actions = bazel_build_args.get_bazel_expanded_actions(
            bazel_launcher=bazel_launcher,
            bazel_execroot=execroot,
            bazel_target=target,
            config_args=config_args,
            filter_mnemonics=filter_mnemonics,
        )

        # 4. Print the results according to format. Use a pager to make
        #    it human-friendly, as the output will always be long.
        import pydoc

        out_lines = []
        if args.format == "json":
            json_actions = [action.to_dict() for action in expanded_actions]
            out_lines.append(json.dumps(json_actions, indent=2))
        else:
            for action in expanded_actions:
                out_lines.append(
                    ExpandCommand._format_human_command(
                        action.target,
                        action.configuration,
                        action.mnemonic,
                        action.command,
                        env_vars=action.env_vars if action.env_vars else None,
                        warnings=action.warnings if action.warnings else None,
                    )
                )
        pydoc.pager("\n".join(out_lines))
        return 0


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--bazel", default="bazel", help="Specify bazel executable"
    )
    parser.add_argument(
        "--buildifier",
        default="buildifier",
        help="Specify buildifier executable",
    )
    parser.add_argument(
        "--workspace", type=Path, help="Specify workspace directory"
    )
    parser.add_argument(
        "--fuchsia-dir", type=Path, help="Specify path to Fuchsia directory"
    )
    parser.add_argument(
        "--build-dir", type=Path, help="Specify path to Ninja build directory"
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        default=0,
        help="Increase verbosity for debugging.",
    )
    parser.add_argument(
        "--quiet", action="count", default=0, help="Decrease verbosity."
    )

    commands = ScriptCommandList(parser)
    commands.add_command(TargetDumpCommand())
    commands.add_command(ActionsCommand())
    commands.add_command(SetGnTargetsCommand())
    commands.add_command(ListBazelHostTestsCommand())
    commands.add_command(ExpandCommand())

    args = parser.parse_args()

    if not args.fuchsia_dir:
        # Assume this script is under //scripts/
        args.fuchsia_dir = Path(__file__).parent.parent.resolve()

    # If --verbose --verbose is used, raise the error, as
    # this is useful when debugging this script to catch
    # AttributeError exceptions that are not caused by a
    # missing command.
    return commands.run(args, keep_exception=args.verbose >= 2)


if __name__ == "__main__":
    sys.exit(main())