blob: 79f312d2208d065666d9fa229bfd9be2d20da5ac [file] [edit]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2025 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Utilities for converting Bazel aspect outputs to rust-project.json format."""
import json
import os
import sys
import typing as T
from pathlib import Path
_SCRIPT_DIR = os.path.dirname(__file__)
sys.path.insert(0, _SCRIPT_DIR)
import build_utils
# Set this to True to debug operations locally in this script.
_DEBUG = False
# Most of the types and logic in this file are re-implementation of
# @rules_rust//tools/rust_analyzer and thus must be updated in lockstep with any
# new rolls of @rules_rust releases into the Fuchsia source checkout.
# LINT.IfChange(rules_rust_version)
# Arguments to pass to Bazel to suppress CLI outputs.
_SILENT_BAZEL_ARGS = [
"--ui_event_filters=-info,-warning",
"--noshow_loading_progress",
"--noshow_progress",
"--show_result=0",
]
# Aspect to use for building/querying rust-analyzer related data from Rust Bazel targets.
_RUST_ANALYZER_ASPECT = "@rules_rust//rust:defs.bzl%rust_analyzer_aspect"
# The list of Bazel output groups to request when using the aspect.
# LINT.IfChange(rust_analyzer_output_groups)
RUST_ANALYZER_OUTPUT_GROUPS = [
"rust_analyzer_crate_spec",
"rust_generated_srcs",
"rust_analyzer_proc_macro_dylib",
"rust_analyzer_src",
"fuchsia_rust_analyzer_manifest",
]
# LINT.ThenChange(//build/bazel/aspects/rust_analyzer.bzl:rust_analyzer_output_groups)
# The prefix that appears in Bazel stderr DEBUG lines, generated by
# the rust_analyzer aspect.
# LINT.IfChange(rust_analyzer_manifest_path_prefix)
FUCHSIA_RUST_ANALYZER_MANIFEST_PATH_PREFIX = (
b"FUCHSIA_RUST_ANALYZER_MANIFEST_PATH="
)
# LINT.ThenChange(//build/bazel/aspects/rust_analyzer.bzl:rust_analyzer_manifest_path_prefix)
class CrateSpecSource(T.TypedDict, total=False):
"""Source file information for a crate, from the aspect."""
exclude_dirs: list[str]
include_dirs: list[str]
class CrateSpecBuild(T.TypedDict, total=False):
"""Build information for a crate, from the aspect."""
label: str
build_file: str
class CrateSpec(T.TypedDict, total=False):
"""Raw crate specification as output by the rust_analyzer_aspect."""
aliases: dict[str, str]
crate_id: str
display_name: str
edition: str
root_module: str
is_workspace_member: bool
deps: list[str] # list of crate_ids
proc_macro_dylib_path: T.Optional[str]
source: T.Optional[CrateSpecSource]
cfg: list[str]
env: dict[str, str]
target: str
crate_type: str # bin, rlib, lib, dylib, cdylib, staticlib, proc-macro
is_test: bool
build: T.Optional[CrateSpecBuild]
def consolidate_crate_specs(crate_specs: list[CrateSpec]) -> list[CrateSpec]:
"""Consolidate a CrateSpec list.
This de-duplicates items with the same crate_id, which happens
when a rust_test() depends on a rust_library(). See
consolidate_crate_spec() in @rules_rust//tools/rust_analyzer:aquery.rs
Args:
crate_specs: A CrateSpec list.
Returns:
A new CrateSpec list.
"""
def extend_str_list(dest: T.Any, key: str, src: T.Any) -> None:
"""Extend dest[key] with the items from src[key] that are not already in the list."""
current_items = set(dest[key])
dest[key].extend(
[item for item in src[key] if item not in current_items]
)
id_to_spec: dict[str, CrateSpec] = {}
for crate in crate_specs:
crate_id = crate["crate_id"]
current = id_to_spec.setdefault(crate_id, crate)
if current == crate:
continue
extend_str_list(current, "deps", crate)
current["env"].update(crate["env"])
current["aliases"].update(crate["aliases"])
current_source = current.get("source")
crate_source = crate.get("source")
if current_source is None:
current["source"] = crate_source
elif crate_source:
extend_str_list(current_source, "include_dirs", crate_source)
extend_str_list(current_source, "exclude_dirs", crate_source)
extend_str_list(current, "cfg", crate)
# display_name should match the library's crate name because Rust Analyzer
# seems to use display_name for matching crate entries in rust-project.json
# against symbols in source files. For more details, see
# https://github.com/bazelbuild/rules_rust/issues/1032
if crate["crate_type"] == "rlib":
current["display_name"] = crate["display_name"]
current["crate_type"] = "rlib"
current["is_test"] = crate["is_test"]
# We want to use the test target's build label to provide
# unit tests codelens actions for library crates in IDEs.
if crate["is_test"]:
crate_build = crate.get("build")
if crate_build:
current["build"] = crate_build
# For proc-macro crates that exist within the workspace, there will be a
# generated crate-spec in both the fastbuild and opt-exec configuration.
# Prefer proc macro paths with an opt-exec component in the path.
crate_dylib_path = crate.get("proc_macro_dylib_path")
if crate_dylib_path:
if "-opt-exec-" in crate_dylib_path:
current["proc_macro_dylib_path"] = crate_dylib_path
return list(id_to_spec.values())
# See rust-project.json format at
# https://rust-analyzer.github.io/book/non_cargo_based_projects.html
class Dependency(T.TypedDict, total=False):
"""Represents a dependency in the rust-project.json format."""
crate: int # Index in the final crates array
name: str
class Source(T.TypedDict, total=False):
"""Source file information in the rust-project.json format."""
include_dirs: list[str]
exclude_dirs: list[str]
class Build(T.TypedDict, total=False):
"""Build information in the rust-project.json format."""
label: str
build_file: str
target_kind: str # bin, lib, test
class Crate(T.TypedDict, total=False):
"""Represents a crate in the rust-project.json format."""
crate_id: int
display_name: T.Optional[str]
root_module: str
edition: str
deps: list[Dependency] # This will be empty in this function
is_workspace_member: T.Optional[bool]
source: T.Optional[Source]
cfg: list[str]
target: T.Optional[str]
env: T.Optional[T.Dict[str, str]]
is_proc_macro: bool
proc_macro_dylib_path: T.Optional[str]
build: T.Optional[Build]
def convert_crate_specs_to_rust_project_crates(
crate_specs: list[CrateSpec],
) -> list[Crate]:
"""
Converts a list of CrateSpec dictionaries to a list of Crate dictionaries.
This function takes the raw crate specifications output by the Bazel aspect
and transforms them into the format expected by rust-analyzer in the
rust-project.json file. It resolves crate dependencies by their `crate_id`
and replaces them with an index into the final list of crates.
Args:
crate_specs: A list of dictionaries, where each dictionary represents
a crate's metadata as produced by the rust_analyzer_aspect.
Returns:
A list of dictionaries, formatted according to the Crate T.TypedDict,
suitable for inclusion in the rust-project.json 'crates' array.
"""
# Sort the crate specs by display name to ensure reproducible results.
# Since several versions of the same crate can be included in a single
# build, use build.label as a secondary key if present since it will
# typically include a version number.
sorted_crate_specs = sorted(
crate_specs,
key=lambda c: (
c["display_name"],
c["build"]["label"] if c.get("build") else "",
),
)
crate_id_to_index: dict[str, int] = {
spec["crate_id"]: i for i, spec in enumerate(sorted_crate_specs)
}
crate_id_to_spec: dict[str, CrateSpec] = {
spec["crate_id"]: spec for spec in sorted_crate_specs
}
result_crates: list[Crate] = []
for crate_spec in sorted_crate_specs:
target_kind = "lib"
crate_type = crate_spec.get("crate_type", "rlib")
is_test = crate_spec.get("is_test", False)
if crate_type == "bin":
target_kind = "test" if is_test else "bin"
source: T.Optional[Source] = None
spec_source = crate_spec.get("source")
if spec_source:
include_dirs = spec_source.get("include_dirs", [])
exclude_dirs = spec_source.get("exclude_dirs", [])
# If both include_dirs and exclude_dirs are empty, we don't include the optional source
# field in the final rust-project.json, so rust-analyzer can use root_module to
# determine the crate's source files.
if include_dirs or exclude_dirs:
source = {
"include_dirs": include_dirs,
"exclude_dirs": exclude_dirs,
}
build: T.Optional[Build] = None
spec_build = crate_spec.get("build")
if spec_build:
build = {
"label": spec_build.get("label", ""),
"build_file": spec_build.get("build_file", ""),
"target_kind": target_kind,
}
deps: list[Dependency] = []
for dep_id in crate_spec.get("deps", []):
if dep_id not in crate_id_to_index:
if _DEBUG:
print(
f"Warning: Dependency '{dep_id}' not found for crate '{crate_spec.get('crate_id')}'",
file=sys.stderr,
)
continue
dep_index = crate_id_to_index[dep_id]
dep_spec = crate_id_to_spec[dep_id]
deps.append({"crate": dep_index, "name": dep_spec["display_name"]})
result_crate: Crate = {
"crate_id": crate_id_to_index[crate_spec["crate_id"]],
"display_name": crate_spec["display_name"],
"root_module": crate_spec["root_module"],
"edition": crate_spec["edition"],
"deps": deps,
"is_workspace_member": crate_spec["is_workspace_member"],
"cfg": crate_spec["cfg"],
"target": crate_spec["target"],
"env": crate_spec["env"],
"is_proc_macro": (
crate_spec.get("proc_macro_dylib_path") is not None
),
"proc_macro_dylib_path": crate_spec.get("proc_macro_dylib_path"),
"build": build,
}
if source:
result_crate["source"] = source
result_crates.append(result_crate)
return result_crates
def substitute_tokens(text: str, bazel_paths: build_utils.BazelPaths) -> str:
"""
Replaces placeholder tokens in a string with actual paths from BazelPaths.
The following substitutions are made:
__WORKSPACE__: Fuchsia source root directory. Note this is intentionally not the Bazel
workspace root directory, so editors can correctly map source files to their locations
in the Fuchsia source tree. These files are symlinked to our synthesized Bazel
workspace.
${pwd}: Bazel execution root directory.
__EXEC_ROOT__: Bazel execution root directory.
__OUTPUT_BASE__: Bazel output base directory.
The substitutions are based on the output from rules_rust.
See more details in https://github.com/bazelbuild/rules_rust/blob/6b4edd077776d719fc3bb4f891f92e782e68fdaa/tools/rust_analyzer/lib.rs#L157
Args:
text: The string containing tokens to be replaced.
bazel_paths: A BazelPaths object containing the relevant paths.
Returns:
The string with all tokens substituted.
"""
return (
text.replace("__WORKSPACE__", str(bazel_paths.fuchsia_dir))
.replace("${pwd}", str(bazel_paths.execroot))
.replace("__EXEC_ROOT__", str(bazel_paths.execroot))
.replace("__OUTPUT_BASE__", str(bazel_paths.output_base))
)
def load_crate_spec_from_json(
file_path: Path, bazel_paths: build_utils.BazelPaths
) -> CrateSpec:
"""
Loads a CrateSpec dictionary from a JSON file, performing token substitutions.
Args:
file_path: Path to the .rust_analyzer_crate_spec.json file.
bazel_paths: A BazelPaths object containing the relevant paths.
Returns:
A CrateSpec dictionary.
"""
return json.loads(substitute_tokens(file_path.read_text(), bazel_paths))
def load_crate_specs_from_json_files(
crate_spec_files: list[Path], bazel_paths: build_utils.BazelPaths
) -> list[CrateSpec]:
"""Load crate specs from JSON files.
Args:
crate_spec_files: list of paths to crate spec JSON files produced by rust_analyzer_aspect.
bazel_paths: A BazelPaths object containing the relevant paths.
Returns:
A list of CrateSpec objects.
"""
crate_specs = [
load_crate_spec_from_json(file_path, bazel_paths)
for file_path in crate_spec_files
]
return consolidate_crate_specs(crate_specs)
def merge_rust_project_jsons(
base_json: dict[str, T.Any], jsons_to_merge: list[dict[str, T.Any]]
) -> dict[str, T.Any]:
"""
Merges multiple rust-project.json structures into a base one.
Deduplicates crates based on (root_module, target) pair.
Renumbers crate IDs in merged JSONs to avoid conflicts with the base JSON.
NOTE: Input dictionaries are modified in-place to avoid copying for efficiency reasons. If this
is a problem, make a copy at the call site.
Args:
base_json: The base rust-project.json dict.
jsons_to_merge: A list of rust-project.json dicts to merge into the base.
Returns:
The base rust-project.json dict with merged crates.
"""
merged_json = base_json
base_crates = merged_json["crates"]
# Create a lookup set for existing crates in the base JSON for deduplication.
# Also map (root_module, target) to the base crate_id for dependency resolution.
existing_crates = {}
base_max_crate_id = -1
for crate in base_crates:
root_module = crate["root_module"]
target = crate["target"]
crate_id = crate["crate_id"]
existing_crates[(root_module, target)] = crate_id
base_max_crate_id = max(base_max_crate_id, crate_id)
current_offset = base_max_crate_id + 1
for json_to_merge in jsons_to_merge:
crates_to_merge = json_to_merge["crates"]
current_json_max_id = -1
# First pass: Identify crates to keep and build a remapping table for this JSON.
id_remap = {}
crates_to_add = []
for crate in crates_to_merge:
root_module = crate["root_module"]
target = crate["target"]
crate_id = crate["crate_id"]
current_json_max_id = max(current_json_max_id, crate_id)
if (root_module, target) in existing_crates:
# If it's a duplicate, map its local ID to the existing base ID.
id_remap[crate_id] = existing_crates[(root_module, target)]
continue
# It's a new crate, needs remapping.
new_id = crate_id + current_offset
id_remap[crate_id] = new_id
# Add to existing_crates so subsequent JSONs also deduplicate against this one.
existing_crates[(root_module, target)] = new_id
crates_to_add.append(crate)
# Second pass: Apply remapping and add to merged list.
for crate in crates_to_add:
crate["crate_id"] = id_remap[crate["crate_id"]]
# Remap dependencies
new_deps = []
for dep in crate.get("deps", []):
original_dep_crate = dep["crate"]
if original_dep_crate in id_remap:
dep["crate"] = id_remap[original_dep_crate]
else:
# If dependency isn't found, simply add the current offset. Although this likely
# means the crate is invalid.
if _DEBUG:
print(
f"Warning: Dependency {original_dep_crate} not found in remapping.",
file=sys.stderr,
)
dep["crate"] = original_dep_crate + current_offset
new_deps.append(dep)
if new_deps:
crate["deps"] = new_deps
merged_json["crates"].append(crate)
# Update offset for the next JSON file to merge.
current_offset += current_json_max_id + 1
return merged_json
def find_crates_for_file(file_path: Path, crates: list[Crate]) -> list[Crate]:
"""
Finds the crates that contains the given Rust file.
This function is best-effort, it returns the crates with a longest
root_module that is a parent of the file_path. If no crate is found, it
returns an empty list.
Args:
file_path: The path to the Rust file.
crates: A list of Crate dictionaries.
Returns:
A list of Crate dictionaries that contain the file.
"""
found = []
foundRootModule = None
for crate in crates:
is_candidate = False
root_module = Path(crate["root_module"])
is_candidate = file_path.is_relative_to(root_module.parent)
source = crate.get("source")
if source and not is_candidate:
for include_dir in source.get("include_dirs", []):
if file_path.is_relative_to(include_dir):
is_candidate = True
break
if source and is_candidate:
for exclude_dir in source.get("exclude_dirs", []):
if file_path.is_relative_to(exclude_dir):
is_candidate = False
break
if not is_candidate:
continue
if foundRootModule and root_module.parent.as_posix() == foundRootModule:
found.append(crate)
elif not foundRootModule or len(root_module.parent.as_posix()) > len(
foundRootModule
):
found = [crate]
foundRootModule = root_module.parent.as_posix()
return found
# LINT.ThenChange(//build/bazel/toplevel.MODULE.bazel:rules_rust_version)
def get_crates_and_dependencies(
interest: list[Crate], crates: list[Crate]
) -> list[Crate]:
"""
Returns a list including the interest crates and all their recursive
dependencies.
Args:
interest: The starting crates.
crates: The full list of crates, used to resolve dependency indices.
Returns:
A list of Crate dictionaries, starting with the input crates, followed
by their dependencies in breadth-first order.
"""
result = interest
visited = {crate["crate_id"] for crate in interest}
# Index crates by crate_id for quick lookup, just in case the crate_id
# used by crates are not guaranteed to be the same as their indices in
# the list.
crate_id_to_crate = {c["crate_id"]: c for c in crates}
for crate in interest:
if not crate["crate_id"] in crate_id_to_crate:
raise ValueError(
f"Crate {crate['crate_id']} not found in crate list."
)
# Make a copy of the deps list to avoid modifying the original crate.
q = [dep for crate in interest for dep in crate.get("deps", [])]
while q:
dep = q.pop(0)
dep_crate_id = dep["crate"]
if dep_crate_id in visited:
continue
if dep_crate_id not in crate_id_to_crate:
raise ValueError(
f"Dependency crate {dep_crate_id} not found in crate list."
)
visited.add(dep_crate_id)
dep_crate = crate_id_to_crate[dep_crate_id]
result.append(dep_crate)
q.extend(dep_crate.get("deps", []))
# Remap crate_ids to be contiguous 0-based indices, as required by
# rust-analyzer. rust-analyzer uses the index in the 'crates' array to
# resolve dependencies, ignoring the 'crate_id' field in the crate object
# itself.
old_id_to_new_index = {c["crate_id"]: i for i, c in enumerate(result)}
remapped_result = []
for i, c in enumerate(result):
# Shallow copy to avoid modifying the input crates.
remapped_crate = c.copy()
remapped_crate["crate_id"] = old_id_to_new_index[c["crate_id"]]
remapped_deps: list[Dependency] = []
for dep in c.get("deps", []):
old_id = dep["crate"]
assert old_id in old_id_to_new_index
remapped_deps.append(
{
"crate": old_id_to_new_index[old_id],
"name": dep["name"],
}
)
remapped_crate["deps"] = remapped_deps
remapped_result.append(remapped_crate)
return remapped_result