rules_rust: enable pipelined compilation. (#1275)
Pipelined compilation allows better parallelism during builds as it
allows libraries to generate lightweight metadata files to unlock other
depencies. These metadata files (.rmeta) can only be used to unlock
library -> library dependencies and do not affect builds in any other
way. This is currently the default in cargo:
https://internals.rust-lang.org/t/evaluating-pipelined-rustc-compilation/10199.
Pipelined compilation will be disabled by default and will need to be
enabled via flag. Pipelined compilation is not supported on windows and will
thus always be disabled.
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index 06d1868..864496d 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -15,6 +15,7 @@
- "-//bindgen/..."
- "-//test/proto/..."
- "-//tools/rust_analyzer/..."
+ - "-//test/unit/pipelined_compilation/..."
crate_universe_vendor_example_targets: &crate_universe_vendor_example_targets
- "//vendor_external:crates_vendor"
- "//vendor_local_manifests:crates_vendor"
diff --git a/docs/flatten.md b/docs/flatten.md
index 725a9fc..87af5c5 100644
--- a/docs/flatten.md
+++ b/docs/flatten.md
@@ -1350,8 +1350,8 @@
## CrateInfo
<pre>
-CrateInfo(<a href="#CrateInfo-aliases">aliases</a>, <a href="#CrateInfo-compile_data">compile_data</a>, <a href="#CrateInfo-deps">deps</a>, <a href="#CrateInfo-edition">edition</a>, <a href="#CrateInfo-is_test">is_test</a>, <a href="#CrateInfo-name">name</a>, <a href="#CrateInfo-output">output</a>, <a href="#CrateInfo-owner">owner</a>, <a href="#CrateInfo-proc_macro_deps">proc_macro_deps</a>, <a href="#CrateInfo-root">root</a>,
- <a href="#CrateInfo-rustc_env">rustc_env</a>, <a href="#CrateInfo-srcs">srcs</a>, <a href="#CrateInfo-type">type</a>, <a href="#CrateInfo-wrapped_crate_type">wrapped_crate_type</a>)
+CrateInfo(<a href="#CrateInfo-aliases">aliases</a>, <a href="#CrateInfo-compile_data">compile_data</a>, <a href="#CrateInfo-deps">deps</a>, <a href="#CrateInfo-edition">edition</a>, <a href="#CrateInfo-is_test">is_test</a>, <a href="#CrateInfo-metadata">metadata</a>, <a href="#CrateInfo-name">name</a>, <a href="#CrateInfo-output">output</a>, <a href="#CrateInfo-owner">owner</a>,
+ <a href="#CrateInfo-proc_macro_deps">proc_macro_deps</a>, <a href="#CrateInfo-root">root</a>, <a href="#CrateInfo-rustc_env">rustc_env</a>, <a href="#CrateInfo-srcs">srcs</a>, <a href="#CrateInfo-type">type</a>, <a href="#CrateInfo-wrapped_crate_type">wrapped_crate_type</a>)
</pre>
A provider containing general Crate information.
@@ -1366,6 +1366,7 @@
| <a id="CrateInfo-deps"></a>deps | depset[DepVariantInfo]: This crate's (rust or cc) dependencies' providers. |
| <a id="CrateInfo-edition"></a>edition | str: The edition of this crate. |
| <a id="CrateInfo-is_test"></a>is_test | bool: If the crate is being compiled in a test context |
+| <a id="CrateInfo-metadata"></a>metadata | File: The rmeta file produced for this crate. It is optional. |
| <a id="CrateInfo-name"></a>name | str: The name of this crate. |
| <a id="CrateInfo-output"></a>output | File: The output File that will be produced, depends on crate type. |
| <a id="CrateInfo-owner"></a>owner | Label: The label of the target that produced this CrateInfo |
@@ -1383,7 +1384,8 @@
<pre>
DepInfo(<a href="#DepInfo-dep_env">dep_env</a>, <a href="#DepInfo-direct_crates">direct_crates</a>, <a href="#DepInfo-link_search_path_files">link_search_path_files</a>, <a href="#DepInfo-transitive_build_infos">transitive_build_infos</a>,
- <a href="#DepInfo-transitive_crate_outputs">transitive_crate_outputs</a>, <a href="#DepInfo-transitive_crates">transitive_crates</a>, <a href="#DepInfo-transitive_noncrates">transitive_noncrates</a>)
+ <a href="#DepInfo-transitive_crate_outputs">transitive_crate_outputs</a>, <a href="#DepInfo-transitive_crates">transitive_crates</a>, <a href="#DepInfo-transitive_metadata_outputs">transitive_metadata_outputs</a>,
+ <a href="#DepInfo-transitive_noncrates">transitive_noncrates</a>)
</pre>
A provider containing information about a Crate's dependencies.
@@ -1399,6 +1401,7 @@
| <a id="DepInfo-transitive_build_infos"></a>transitive_build_infos | depset[BuildInfo] |
| <a id="DepInfo-transitive_crate_outputs"></a>transitive_crate_outputs | depset[File]: All transitive crate outputs. |
| <a id="DepInfo-transitive_crates"></a>transitive_crates | depset[CrateInfo] |
+| <a id="DepInfo-transitive_metadata_outputs"></a>transitive_metadata_outputs | depset[File]: All transitive metadata dependencies (.rmeta, for crates that provide them) and all transitive object dependencies (.rlib) for crates that don't provide metadata. |
| <a id="DepInfo-transitive_noncrates"></a>transitive_noncrates | depset[LinkerInput]: All transitive dependencies that aren't crates. |
diff --git a/docs/providers.md b/docs/providers.md
index 884a154..775dbd9 100644
--- a/docs/providers.md
+++ b/docs/providers.md
@@ -10,8 +10,8 @@
## CrateInfo
<pre>
-CrateInfo(<a href="#CrateInfo-aliases">aliases</a>, <a href="#CrateInfo-compile_data">compile_data</a>, <a href="#CrateInfo-deps">deps</a>, <a href="#CrateInfo-edition">edition</a>, <a href="#CrateInfo-is_test">is_test</a>, <a href="#CrateInfo-name">name</a>, <a href="#CrateInfo-output">output</a>, <a href="#CrateInfo-owner">owner</a>, <a href="#CrateInfo-proc_macro_deps">proc_macro_deps</a>, <a href="#CrateInfo-root">root</a>,
- <a href="#CrateInfo-rustc_env">rustc_env</a>, <a href="#CrateInfo-srcs">srcs</a>, <a href="#CrateInfo-type">type</a>, <a href="#CrateInfo-wrapped_crate_type">wrapped_crate_type</a>)
+CrateInfo(<a href="#CrateInfo-aliases">aliases</a>, <a href="#CrateInfo-compile_data">compile_data</a>, <a href="#CrateInfo-deps">deps</a>, <a href="#CrateInfo-edition">edition</a>, <a href="#CrateInfo-is_test">is_test</a>, <a href="#CrateInfo-metadata">metadata</a>, <a href="#CrateInfo-name">name</a>, <a href="#CrateInfo-output">output</a>, <a href="#CrateInfo-owner">owner</a>,
+ <a href="#CrateInfo-proc_macro_deps">proc_macro_deps</a>, <a href="#CrateInfo-root">root</a>, <a href="#CrateInfo-rustc_env">rustc_env</a>, <a href="#CrateInfo-srcs">srcs</a>, <a href="#CrateInfo-type">type</a>, <a href="#CrateInfo-wrapped_crate_type">wrapped_crate_type</a>)
</pre>
A provider containing general Crate information.
@@ -26,6 +26,7 @@
| <a id="CrateInfo-deps"></a>deps | depset[DepVariantInfo]: This crate's (rust or cc) dependencies' providers. |
| <a id="CrateInfo-edition"></a>edition | str: The edition of this crate. |
| <a id="CrateInfo-is_test"></a>is_test | bool: If the crate is being compiled in a test context |
+| <a id="CrateInfo-metadata"></a>metadata | File: The rmeta file produced for this crate. It is optional. |
| <a id="CrateInfo-name"></a>name | str: The name of this crate. |
| <a id="CrateInfo-output"></a>output | File: The output File that will be produced, depends on crate type. |
| <a id="CrateInfo-owner"></a>owner | Label: The label of the target that produced this CrateInfo |
@@ -43,7 +44,8 @@
<pre>
DepInfo(<a href="#DepInfo-dep_env">dep_env</a>, <a href="#DepInfo-direct_crates">direct_crates</a>, <a href="#DepInfo-link_search_path_files">link_search_path_files</a>, <a href="#DepInfo-transitive_build_infos">transitive_build_infos</a>,
- <a href="#DepInfo-transitive_crate_outputs">transitive_crate_outputs</a>, <a href="#DepInfo-transitive_crates">transitive_crates</a>, <a href="#DepInfo-transitive_noncrates">transitive_noncrates</a>)
+ <a href="#DepInfo-transitive_crate_outputs">transitive_crate_outputs</a>, <a href="#DepInfo-transitive_crates">transitive_crates</a>, <a href="#DepInfo-transitive_metadata_outputs">transitive_metadata_outputs</a>,
+ <a href="#DepInfo-transitive_noncrates">transitive_noncrates</a>)
</pre>
A provider containing information about a Crate's dependencies.
@@ -59,6 +61,7 @@
| <a id="DepInfo-transitive_build_infos"></a>transitive_build_infos | depset[BuildInfo] |
| <a id="DepInfo-transitive_crate_outputs"></a>transitive_crate_outputs | depset[File]: All transitive crate outputs. |
| <a id="DepInfo-transitive_crates"></a>transitive_crates | depset[CrateInfo] |
+| <a id="DepInfo-transitive_metadata_outputs"></a>transitive_metadata_outputs | depset[File]: All transitive metadata dependencies (.rmeta, for crates that provide them) and all transitive object dependencies (.rlib) for crates that don't provide metadata. |
| <a id="DepInfo-transitive_noncrates"></a>transitive_noncrates | depset[LinkerInput]: All transitive dependencies that aren't crates. |
diff --git a/proto/proto.bzl b/proto/proto.bzl
index 9cb83f3..b5e65a3 100644
--- a/proto/proto.bzl
+++ b/proto/proto.bzl
@@ -44,7 +44,7 @@
load("//rust/private:rustc.bzl", "rustc_compile_action")
# buildifier: disable=bzl-visibility
-load("//rust/private:utils.bzl", "compute_crate_name", "determine_output_hash", "find_toolchain", "transform_deps")
+load("//rust/private:utils.bzl", "can_build_metadata", "compute_crate_name", "determine_output_hash", "find_toolchain", "transform_deps")
RustProtoInfo = provider(
doc = "Rust protobuf provider info",
@@ -212,6 +212,13 @@
crate_name,
output_hash,
))
+ rust_metadata = None
+ if can_build_metadata(toolchain, ctx, "rlib"):
+ rust_metadata = ctx.actions.declare_file("%s/lib%s-%s.rmeta" % (
+ output_dir,
+ crate_name,
+ output_hash,
+ ))
# Gather all dependencies for compilation
compile_action_deps = depset(
@@ -234,6 +241,7 @@
proc_macro_deps = depset([]),
aliases = {},
output = rust_lib,
+ metadata = rust_metadata,
edition = proto_toolchain.edition,
rustc_env = {},
is_test = False,
diff --git a/rust/private/common.bzl b/rust/private/common.bzl
index 1cf84cb..3de7cb5 100644
--- a/rust/private/common.bzl
+++ b/rust/private/common.bzl
@@ -47,6 +47,8 @@
"""
if not "wrapped_crate_type" in kwargs:
kwargs.update({"wrapped_crate_type": None})
+ if not "metadata" in kwargs:
+ kwargs.update({"metadata": None})
return CrateInfo(**kwargs)
rust_common = struct(
diff --git a/rust/private/providers.bzl b/rust/private/providers.bzl
index 0a1d924..7533349 100644
--- a/rust/private/providers.bzl
+++ b/rust/private/providers.bzl
@@ -22,6 +22,7 @@
"deps": "depset[DepVariantInfo]: This crate's (rust or cc) dependencies' providers.",
"edition": "str: The edition of this crate.",
"is_test": "bool: If the crate is being compiled in a test context",
+ "metadata": "File: The rmeta file produced for this crate. It is optional.",
"name": "str: The name of this crate.",
"output": "File: The output File that will be produced, depends on crate type.",
"owner": "Label: The label of the target that produced this CrateInfo",
@@ -49,6 +50,7 @@
"transitive_build_infos": "depset[BuildInfo]",
"transitive_crate_outputs": "depset[File]: All transitive crate outputs.",
"transitive_crates": "depset[CrateInfo]",
+ "transitive_metadata_outputs": "depset[File]: All transitive metadata dependencies (.rmeta, for crates that provide them) and all transitive object dependencies (.rlib) for crates that don't provide metadata.",
"transitive_noncrates": "depset[LinkerInput]: All transitive dependencies that aren't crates.",
},
)
diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl
index f50303e..c6361bd 100644
--- a/rust/private/rust.bzl
+++ b/rust/private/rust.bzl
@@ -13,10 +13,12 @@
# limitations under the License.
# buildifier: disable=module-docstring
+load("@bazel_skylib//lib:paths.bzl", "paths")
load("//rust/private:common.bzl", "rust_common")
load("//rust/private:rustc.bzl", "rustc_compile_action")
load(
"//rust/private:utils.bzl",
+ "can_build_metadata",
"compute_crate_name",
"dedent",
"determine_output_hash",
@@ -25,7 +27,6 @@
"get_import_macro_deps",
"transform_deps",
)
-
# TODO(marco): Separate each rule into its own file.
def _assert_no_deprecated_attributes(_ctx):
@@ -316,6 +317,13 @@
)
rust_lib = ctx.actions.declare_file(rust_lib_name)
+ rust_metadata = None
+ if can_build_metadata(toolchain, ctx, crate_type):
+ rust_metadata = ctx.actions.declare_file(
+ paths.replace_extension(rust_lib_name, ".rmeta"),
+ sibling = rust_lib,
+ )
+
deps = transform_deps(ctx.attr.deps)
proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx))
@@ -332,6 +340,7 @@
proc_macro_deps = depset(proc_macro_deps),
aliases = ctx.attr.aliases,
output = rust_lib,
+ metadata = rust_metadata,
edition = get_edition(ctx.attr, toolchain, ctx.label),
rustc_env = ctx.attr.rustc_env,
is_test = False,
diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl
index 21fb46c..0fe072b 100644
--- a/rust/private/rustc.bzl
+++ b/rust/private/rustc.bzl
@@ -169,6 +169,9 @@
return cc_toolchain.needs_pic_for_dynamic_libraries(feature_configuration = feature_configuration)
return False
+def _is_proc_macro(crate_info):
+ return "proc-macro" in (crate_info.type, crate_info.wrapped_crate_type)
+
def collect_deps(
deps,
proc_macro_deps,
@@ -197,6 +200,7 @@
build_info = None
linkstamps = []
transitive_crate_outputs = []
+ transitive_metadata_outputs = []
aliases = {k.label: v for k, v in aliases.items()}
for dep in depset(transitive = [deps, proc_macro_deps]).to_list():
@@ -222,19 +226,31 @@
transitive_crates.append(
depset(
[crate_info],
- transitive = [] if "proc-macro" in [
- crate_info.type,
- crate_info.wrapped_crate_type,
- ] else [dep_info.transitive_crates],
+ transitive = [] if _is_proc_macro(crate_info) else [dep_info.transitive_crates],
),
)
+
+ # If this dependency produces metadata, add it to the metadata outputs.
+ # If it doesn't (for example a custom library that exports crate_info),
+ # we depend on crate_info.output.
+ depend_on = crate_info.metadata
+ if not crate_info.metadata:
+ depend_on = crate_info.output
+
+ # If this dependency is a proc_macro, it still can be used for lib crates
+ # that produce metadata.
+ # In that case, we don't depend on its metadata dependencies.
+ transitive_metadata_outputs.append(
+ depset(
+ [depend_on],
+ transitive = [] if _is_proc_macro(crate_info) else [dep_info.transitive_metadata_outputs],
+ ),
+ )
+
transitive_crate_outputs.append(
depset(
[crate_info.output],
- transitive = [] if "proc-macro" in [
- crate_info.type,
- crate_info.wrapped_crate_type,
- ] else [dep_info.transitive_crate_outputs],
+ transitive = [] if _is_proc_macro(crate_info) else [dep_info.transitive_crate_outputs],
),
)
@@ -269,6 +285,7 @@
order = "topological", # dylib link flag ordering matters.
),
transitive_crate_outputs = depset(transitive = transitive_crate_outputs),
+ transitive_metadata_outputs = depset(transitive = transitive_metadata_outputs),
transitive_build_infos = depset(transitive = transitive_build_infos),
link_search_path_files = depset(transitive = transitive_link_search_paths),
dep_env = build_info.dep_env if build_info else None,
@@ -505,6 +522,28 @@
visited_libs[name] = artifact
return ambiguous_libs
+def _depend_on_metadata(crate_info, force_depend_on_objects):
+ """Determines if we can depend on metadata for this crate.
+
+ By default (when pipelining is disabled or when the crate type needs to link against
+ objects) we depend on the set of object files (.rlib).
+ When pipelining is enabled and the crate type supports depending on metadata,
+ we depend on metadata files only (.rmeta).
+ In some rare cases, even if both of those conditions are true, we still want to
+ depend on objects. This is what force_depend_on_objects is.
+
+ Args:
+ crate_info (CrateInfo): The Crate to determine this for.
+ force_depend_on_objects (bool): if set we will not depend on metadata.
+
+ Returns:
+ Whether we can depend on metadata for this crate.
+ """
+ if force_depend_on_objects:
+ return False
+
+ return crate_info.type in ("rlib", "lib")
+
def collect_inputs(
ctx,
file,
@@ -516,7 +555,8 @@
crate_info,
dep_info,
build_info,
- stamp = False):
+ stamp = False,
+ force_depend_on_objects = False):
"""Gather's the inputs and required input information for a rustc action
Args:
@@ -532,6 +572,8 @@
build_info (BuildInfo): The target Crate's build settings.
stamp (bool, optional): Whether or not workspace status stamping is enabled. For more details see
https://docs.bazel.build/versions/main/user-manual.html#flag--stamp
+ force_depend_on_objects (bool, optional): Forces dependencies of this rule to be objects rather than
+ metadata, even for libraries. This is used in rustdoc tests.
Returns:
tuple: A tuple: A tuple of the following items:
@@ -572,6 +614,10 @@
# change.
linkstamp_outs = []
+ transitive_crate_outputs = dep_info.transitive_crate_outputs
+ if _depend_on_metadata(crate_info, force_depend_on_objects):
+ transitive_crate_outputs = dep_info.transitive_metadata_outputs
+
nolinkstamp_compile_inputs = depset(
getattr(files, "data", []) +
([build_info.rustc_env, build_info.flags] if build_info else []) +
@@ -580,7 +626,7 @@
transitive = [
linker_depset,
crate_info.srcs,
- dep_info.transitive_crate_outputs,
+ transitive_crate_outputs,
depset(additional_transitive_inputs),
crate_info.compile_data,
toolchain.all_files,
@@ -654,7 +700,10 @@
force_all_deps_direct = False,
force_link = False,
stamp = False,
- remap_path_prefix = "."):
+ remap_path_prefix = ".",
+ use_json_output = False,
+ build_metadata = False,
+ force_depend_on_objects = False):
"""Builds an Args object containing common rustc flags
Args:
@@ -681,6 +730,9 @@
stamp (bool, optional): Whether or not workspace status stamping is enabled. For more details see
https://docs.bazel.build/versions/main/user-manual.html#flag--stamp
remap_path_prefix (str, optional): A value used to remap `${pwd}` to. If set to a falsey value, no prefix will be set.
+ use_json_output (bool): Have rustc emit json and process_wrapper parse json messages to output rendered output.
+ build_metadata (bool): Generate CLI arguments for building *only* .rmeta files. This requires use_json_output.
+ force_depend_on_objects (bool): Force using `.rlib` object files instead of metadata (`.rmeta`) files even if they are available.
Returns:
tuple: A tuple of the following items
@@ -692,6 +744,9 @@
This is to be passed to the `arguments` parameter of actions
- (dict): Common rustc environment variables
"""
+ if build_metadata and not use_json_output:
+ fail("build_metadata requires parse_json_output")
+
output_dir = getattr(crate_info.output, "dirname", None)
linker_script = getattr(file, "linker_script", None)
@@ -761,8 +816,35 @@
rustc_flags.add(crate_info.root)
rustc_flags.add("--crate-name=" + crate_info.name)
rustc_flags.add("--crate-type=" + crate_info.type)
+
+ error_format = "human"
if hasattr(attr, "_error_format"):
- rustc_flags.add("--error-format=" + attr._error_format[ErrorFormatInfo].error_format)
+ error_format = attr._error_format[ErrorFormatInfo].error_format
+
+ if use_json_output:
+ # If --error-format was set to json, we just pass the output through
+ # Otherwise process_wrapper uses the "rendered" field.
+ process_wrapper_flags.add("--rustc-output-format", "json" if error_format == "json" else "rendered")
+
+ # Configure rustc json output by adding artifact notifications.
+ # These will always be filtered out by process_wrapper and will be use to terminate
+ # rustc when appropriate.
+ json = ["artifacts"]
+ if error_format == "short":
+ json.append("diagnostic-short")
+ elif error_format == "human" and toolchain.os != "windows":
+ # If the os is not windows, we can get colorized output.
+ json.append("diagnostic-rendered-ansi")
+
+ rustc_flags.add("--json=" + ",".join(json))
+
+ error_format = "json"
+
+ if build_metadata:
+ # Configure process_wrapper to terminate rustc when metadata are emitted
+ process_wrapper_flags.add("--rustc-quit-on-rmeta", "true")
+
+ rustc_flags.add("--error-format=" + error_format)
# Mangle symbols to disambiguate crates with the same name. This could
# happen only for non-final artifacts where we compute an output_hash,
@@ -789,7 +871,9 @@
if emit:
rustc_flags.add("--emit=" + ",".join(emit_with_paths))
- rustc_flags.add("--color=always")
+ if error_format != "json":
+ # Color is not compatible with json output.
+ rustc_flags.add("--color=always")
rustc_flags.add("--target=" + toolchain.target_flag_value)
if hasattr(attr, "crate_features"):
rustc_flags.add_all(getattr(attr, "crate_features"), before_each = "--cfg", format_each = 'feature="%s"')
@@ -832,11 +916,12 @@
_add_native_link_flags(rustc_flags, dep_info, linkstamp_outs, ambiguous_libs, crate_info.type, toolchain, cc_toolchain, feature_configuration)
- # These always need to be added, even if not linking this crate.
- add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct)
+ use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects)
- needs_extern_proc_macro_flag = "proc-macro" in [crate_info.type, crate_info.wrapped_crate_type] and \
- crate_info.edition != "2015"
+ # These always need to be added, even if not linking this crate.
+ add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct, use_metadata)
+
+ needs_extern_proc_macro_flag = _is_proc_macro(crate_info) and crate_info.edition != "2015"
if needs_extern_proc_macro_flag:
rustc_flags.add("--extern")
rustc_flags.add("proc_macro")
@@ -919,6 +1004,8 @@
- (DepInfo): The transitive dependencies of this crate.
- (DefaultInfo): The output file for this crate, and its runfiles.
"""
+ build_metadata = getattr(crate_info, "metadata", None)
+
cc_toolchain, feature_configuration = find_cc_toolchain(ctx)
dep_info, build_info, linkstamps = collect_deps(
@@ -948,6 +1035,14 @@
stamp = stamp,
)
+ # If we build metadata, we need to keep the command line of the two invocations
+ # (rlib and rmeta) as similar as possible, otherwise rustc rejects the rmeta as
+ # a candidate.
+ # Because of that we need to add emit=metadata to both the rlib and rmeta invocation.
+ emit = ["dep-info", "link"]
+ if build_metadata:
+ emit.append("metadata")
+
args, env_from_args = construct_arguments(
ctx = ctx,
attr = attr,
@@ -955,6 +1050,7 @@
toolchain = toolchain,
tool_path = toolchain.rustc.path,
cc_toolchain = cc_toolchain,
+ emit = emit,
feature_configuration = feature_configuration,
crate_info = crate_info,
dep_info = dep_info,
@@ -967,8 +1063,35 @@
build_flags_files = build_flags_files,
force_all_deps_direct = force_all_deps_direct,
stamp = stamp,
+ use_json_output = bool(build_metadata),
)
+ args_metadata = None
+ if build_metadata:
+ args_metadata, _ = construct_arguments(
+ ctx = ctx,
+ attr = attr,
+ file = ctx.file,
+ toolchain = toolchain,
+ tool_path = toolchain.rustc.path,
+ cc_toolchain = cc_toolchain,
+ emit = emit,
+ feature_configuration = feature_configuration,
+ crate_info = crate_info,
+ dep_info = dep_info,
+ linkstamp_outs = linkstamp_outs,
+ ambiguous_libs = ambiguous_libs,
+ output_hash = output_hash,
+ rust_flags = rust_flags,
+ out_dir = out_dir,
+ build_env_files = build_env_files,
+ build_flags_files = build_flags_files,
+ force_all_deps_direct = force_all_deps_direct,
+ stamp = stamp,
+ use_json_output = True,
+ build_metadata = True,
+ )
+
env = dict(ctx.configuration.default_shell_env)
env.update(env_from_args)
@@ -1019,10 +1142,25 @@
len(crate_info.srcs.to_list()),
),
)
+ if args_metadata:
+ ctx.actions.run(
+ executable = ctx.executable._process_wrapper,
+ inputs = compile_inputs,
+ outputs = [build_metadata],
+ env = env,
+ arguments = args_metadata.all,
+ mnemonic = "RustcMetadata",
+ progress_message = "Compiling Rust metadata {} {}{} ({} files)".format(
+ crate_info.type,
+ ctx.label.name,
+ formatted_version,
+ len(crate_info.srcs.to_list()),
+ ),
+ )
else:
# Run without process_wrapper
- if build_env_files or build_flags_files or stamp:
- fail("build_env_files, build_flags_files, stamp are not supported when building without process_wrapper")
+ if build_env_files or build_flags_files or stamp or build_metadata:
+ fail("build_env_files, build_flags_files, stamp, build_metadata are not supported when building without process_wrapper")
ctx.actions.run(
executable = toolchain.rustc,
inputs = compile_inputs,
@@ -1304,7 +1442,7 @@
dirs[f.dirname] = None
return dirs.keys()
-def add_crate_link_flags(args, dep_info, force_all_deps_direct = False):
+def add_crate_link_flags(args, dep_info, force_all_deps_direct = False, use_metadata = False):
"""Adds link flags to an Args object reference
Args:
@@ -1312,22 +1450,19 @@
dep_info (DepInfo): The current target's dependency info
force_all_deps_direct (bool, optional): Whether to pass the transitive rlibs with --extern
to the commandline as opposed to -L.
+ use_metadata (bool, optional): Build command line arugments using metadata for crates that provide it.
"""
- if force_all_deps_direct:
- args.add_all(
- depset(
- transitive = [
- dep_info.direct_crates,
- dep_info.transitive_crates,
- ],
- ),
- uniquify = True,
- map_each = _crate_to_link_flag,
- )
- else:
- # nb. Direct crates are linked via --extern regardless of their crate_type
- args.add_all(dep_info.direct_crates, map_each = _crate_to_link_flag)
+ direct_crates = depset(
+ transitive = [
+ dep_info.direct_crates,
+ dep_info.transitive_crates,
+ ],
+ ) if force_all_deps_direct else dep_info.direct_crates
+
+ crate_to_link_flags = _crate_to_link_flag_metadata if use_metadata else _crate_to_link_flag
+ args.add_all(direct_crates, uniquify = True, map_each = crate_to_link_flags)
+
args.add_all(
dep_info.transitive_crates,
map_each = _get_crate_dirname,
@@ -1335,6 +1470,29 @@
format_each = "-Ldependency=%s",
)
+def _crate_to_link_flag_metadata(crate):
+ """A helper macro used by `add_crate_link_flags` for adding crate link flags to a Arg object
+
+ Args:
+ crate (CrateInfo|AliasableDepInfo): A CrateInfo or an AliasableDepInfo provider
+
+ Returns:
+ list: Link flags for the given provider
+ """
+
+ # This is AliasableDepInfo, we should use the alias as a crate name
+ if hasattr(crate, "dep"):
+ name = crate.name
+ crate_info = crate.dep
+ else:
+ name = crate.name
+ crate_info = crate
+
+ lib_or_meta = crate_info.metadata
+ if not crate_info.metadata:
+ lib_or_meta = crate_info.output
+ return ["--extern={}={}".format(name, lib_or_meta.path)]
+
def _crate_to_link_flag(crate):
"""A helper macro used by `add_crate_link_flags` for adding crate link flags to a Arg object
diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl
index 82fdd4e..6874717 100644
--- a/rust/private/rustdoc.bzl
+++ b/rust/private/rustdoc.bzl
@@ -37,6 +37,7 @@
aliases = crate_info.aliases,
# This crate info should have no output
output = None,
+ metadata = None,
edition = crate_info.edition,
rustc_env = crate_info.rustc_env,
is_test = crate_info.is_test,
@@ -90,6 +91,8 @@
crate_info = crate_info,
dep_info = dep_info,
build_info = build_info,
+ # If this is a rustdoc test, we need to depend on rlibs rather than .rmeta.
+ force_depend_on_objects = is_test,
)
# Since this crate is not actually producing the output described by the
@@ -118,6 +121,7 @@
emit = [],
remap_path_prefix = None,
force_link = True,
+ force_depend_on_objects = is_test,
)
# Because rustdoc tests compile tests outside of the sandbox, the sysroot
diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl
index 4ae5b5f..633c90a 100644
--- a/rust/private/utils.bzl
+++ b/rust/private/utils.bzl
@@ -610,3 +610,25 @@
string = string[:pattern_start] + replacement + string[after_pattern:]
return string
+
+def can_build_metadata(toolchain, ctx, crate_type):
+ """Can we build metadata for this rust_library?
+
+ Args:
+ toolchain (toolchain): The rust toolchain
+ ctx (ctx): The rule's context object
+ crate_type (String): one of lib|rlib|dylib|staticlib|cdylib|proc-macro
+
+ Returns:
+ bool: whether we can build metadata for this rule.
+ """
+
+ # In order to enable pipelined compilation we require that:
+ # 1) The _pipelined_compilation flag is enabled,
+ # 2) the OS running the rule is something other than windows as we require sandboxing (for now),
+ # 3) process_wrapper is enabled (this is disabled when compiling process_wrapper itself),
+ # 4) the crate_type is rlib or lib.
+ return toolchain._pipelined_compilation and \
+ toolchain.os != "windows" and \
+ ctx.attr._process_wrapper and \
+ crate_type in ("rlib", "lib")
diff --git a/rust/settings/BUILD.bazel b/rust/settings/BUILD.bazel
index 2ff9a43..c5928a1 100644
--- a/rust/settings/BUILD.bazel
+++ b/rust/settings/BUILD.bazel
@@ -29,6 +29,14 @@
build_setting_default = False,
)
+# When set, this flag causes rustc to emit .rmeta files and use them for rlib -> rlib dependencies.
+# While this involves one extra (short) rustc invocation to build the rmeta file,
+# it allows library dependencies to be unlocked much sooner, increasing parallelism during compilation.
+bool_flag(
+ name = "pipelined_compilation",
+ build_setting_default = False,
+)
+
bzl_library(
name = "bzl_lib",
srcs = glob(["**/*.bzl"]),
diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl
index a046c4e..90a3c59 100644
--- a/rust/toolchain.bzl
+++ b/rust/toolchain.bzl
@@ -426,6 +426,7 @@
rename_first_party_crates = ctx.attr._rename_first_party_crates[BuildSettingInfo].value
third_party_dir = ctx.attr._third_party_dir[BuildSettingInfo].value
+ pipelined_compilation = ctx.attr._pipelined_compilation[BuildSettingInfo].value
if ctx.attr.rust_lib:
# buildifier: disable=print
@@ -536,6 +537,7 @@
# Experimental and incompatible flags
_rename_first_party_crates = rename_first_party_crates,
_third_party_dir = third_party_dir,
+ _pipelined_compilation = pipelined_compilation,
)
return [
toolchain,
@@ -673,6 +675,9 @@
"_cc_toolchain": attr.label(
default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
),
+ "_pipelined_compilation": attr.label(
+ default = "@rules_rust//rust/settings:pipelined_compilation",
+ ),
"_rename_first_party_crates": attr.label(
default = Label("//rust/settings:rename_first_party_crates"),
),
diff --git a/test/process_wrapper/rustc_quit_on_rmeta.rs b/test/process_wrapper/rustc_quit_on_rmeta.rs
index df32341..5559508 100644
--- a/test/process_wrapper/rustc_quit_on_rmeta.rs
+++ b/test/process_wrapper/rustc_quit_on_rmeta.rs
@@ -6,8 +6,8 @@
use runfiles::Runfiles;
- // fake_rustc runs the fake_rustc binary under process_wrapper with the specified
- // process wrapper arguments. No arguments are passed to fake_rustc itself.
+ /// fake_rustc runs the fake_rustc binary under process_wrapper with the specified
+ /// process wrapper arguments. No arguments are passed to fake_rustc itself.
fn fake_rustc(process_wrapper_args: &[&'static str]) -> String {
let r = Runfiles::create().unwrap();
let fake_rustc = r.rlocation(
@@ -59,7 +59,12 @@
#[test]
fn test_rustc_quit_on_rmeta_quits() {
- let out_content = fake_rustc(&["--rustc-quit-on-rmeta", "true"]);
+ let out_content = fake_rustc(&[
+ "--rustc-quit-on-rmeta",
+ "true",
+ "--rustc-output-format",
+ "rendered",
+ ]);
assert!(
!out_content.contains("should not be in output"),
"output should not contain 'should not be in output' but did: {}",
diff --git a/test/unit/pipelined_compilation/BUILD.bazel b/test/unit/pipelined_compilation/BUILD.bazel
new file mode 100644
index 0000000..8d363e0
--- /dev/null
+++ b/test/unit/pipelined_compilation/BUILD.bazel
@@ -0,0 +1,4 @@
+load(":pipelined_compilation_test.bzl", "pipelined_compilation_test_suite")
+
+############################ UNIT TESTS #############################
+pipelined_compilation_test_suite(name = "pipelined_compilation_test_suite")
diff --git a/test/unit/pipelined_compilation/bin.rs b/test/unit/pipelined_compilation/bin.rs
new file mode 100644
index 0000000..aa32dd2
--- /dev/null
+++ b/test/unit/pipelined_compilation/bin.rs
@@ -0,0 +1,5 @@
+use second::fun;
+
+fn main() {
+ fun()
+}
diff --git a/test/unit/pipelined_compilation/custom_rule_test/to_wrap.rs b/test/unit/pipelined_compilation/custom_rule_test/to_wrap.rs
new file mode 100644
index 0000000..5fee30b
--- /dev/null
+++ b/test/unit/pipelined_compilation/custom_rule_test/to_wrap.rs
@@ -0,0 +1,3 @@
+pub fn to_wrap() {
+ eprintln!("something");
+}
diff --git a/test/unit/pipelined_compilation/custom_rule_test/uses_wrapper.rs b/test/unit/pipelined_compilation/custom_rule_test/uses_wrapper.rs
new file mode 100644
index 0000000..d932467
--- /dev/null
+++ b/test/unit/pipelined_compilation/custom_rule_test/uses_wrapper.rs
@@ -0,0 +1,5 @@
+use wrapper::wrap;
+
+pub fn calls_wrap() {
+ wrap();
+}
diff --git a/test/unit/pipelined_compilation/first.rs b/test/unit/pipelined_compilation/first.rs
new file mode 100644
index 0000000..30c0129
--- /dev/null
+++ b/test/unit/pipelined_compilation/first.rs
@@ -0,0 +1,4 @@
+pub fn first_fun() -> u8 {
+ 4 // chosen by fair dice roll.
+ // guaranteed to be random.
+}
diff --git a/test/unit/pipelined_compilation/my_macro.rs b/test/unit/pipelined_compilation/my_macro.rs
new file mode 100644
index 0000000..035c761
--- /dev/null
+++ b/test/unit/pipelined_compilation/my_macro.rs
@@ -0,0 +1,6 @@
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn noop(_attr: TokenStream, item: TokenStream) -> TokenStream {
+ item
+}
diff --git a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl
new file mode 100644
index 0000000..93382c5
--- /dev/null
+++ b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl
@@ -0,0 +1,231 @@
+"""Unittests for rust rules."""
+
+load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts")
+load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro")
+load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements", "assert_list_contains_adjacent_elements_not")
+load(":wrap.bzl", "wrap")
+
+NOT_WINDOWS = select({
+ "@platforms//os:linux": [],
+ "@platforms//os:macos": [],
+ "//conditions:default": ["@platforms//:incompatible"],
+})
+
+ENABLE_PIPELINING = {
+ "@//rust/settings:pipelined_compilation": True,
+}
+
+def _second_lib_test_impl(ctx):
+ env = analysistest.begin(ctx)
+ tut = analysistest.target_under_test(env)
+ rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0]
+ metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0]
+
+ # Both actions should use the same --emit=
+ assert_argv_contains(env, rlib_action, "--emit=dep-info,link,metadata")
+ assert_argv_contains(env, metadata_action, "--emit=dep-info,link,metadata")
+
+ # The metadata action should have a .rmeta as output and the rlib action a .rlib
+ path = rlib_action.outputs.to_list()[0].path
+ asserts.true(
+ env,
+ path.endswith(".rlib"),
+ "expected Rustc to output .rlib, got " + path,
+ )
+ path = metadata_action.outputs.to_list()[0].path
+ asserts.true(
+ env,
+ path.endswith(".rmeta"),
+ "expected RustcMetadata to output .rmeta, got " + path,
+ )
+
+ # Only the action building metadata should contain --rustc-quit-on-rmeta
+ assert_list_contains_adjacent_elements_not(env, rlib_action.argv, ["--rustc-quit-on-rmeta", "true"])
+ assert_list_contains_adjacent_elements(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"])
+
+ # Check that both actions refer to the metadata of :first, not the rlib
+ extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")]
+ asserts.true(
+ env,
+ len(extern_metadata) == 1,
+ "did not find a --extern=first=*.rmeta but expected one",
+ )
+ extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")]
+ asserts.true(
+ env,
+ len(extern_rlib) == 1,
+ "did not find a --extern=first=*.rlib but expected one",
+ )
+
+ # Check that the input to both actions is the metadata of :first
+ input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")]
+ asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input, found " + str([i.path for i in input_metadata]))
+ asserts.true(env, input_metadata[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_metadata[0].path)
+ input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")]
+ asserts.true(env, len(input_rlib) == 1, "expected only one libfirst input, found " + str([i.path for i in input_rlib]))
+ asserts.true(env, input_rlib[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_rlib[0].path)
+
+ return analysistest.end(env)
+
+def _bin_test_impl(ctx):
+ env = analysistest.begin(ctx)
+ tut = analysistest.target_under_test(env)
+ bin_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0]
+
+ # Check that no inputs to this binary are .rmeta files.
+ metadata_inputs = [i.path for i in bin_action.inputs.to_list() if i.path.endswith(".rmeta")]
+ asserts.false(env, metadata_inputs, "expected no metadata inputs, found " + str(metadata_inputs))
+
+ return analysistest.end(env)
+
+bin_test = analysistest.make(_bin_test_impl, config_settings = ENABLE_PIPELINING)
+second_lib_test = analysistest.make(_second_lib_test_impl, config_settings = ENABLE_PIPELINING)
+
+def _pipelined_compilation_test():
+ rust_proc_macro(
+ name = "my_macro",
+ edition = "2021",
+ srcs = ["my_macro.rs"],
+ )
+
+ rust_library(
+ name = "first",
+ edition = "2021",
+ srcs = ["first.rs"],
+ )
+
+ rust_library(
+ name = "second",
+ edition = "2021",
+ srcs = ["second.rs"],
+ deps = [":first"],
+ proc_macro_deps = [":my_macro"],
+ )
+
+ rust_binary(
+ name = "bin",
+ edition = "2021",
+ srcs = ["bin.rs"],
+ deps = [":second"],
+ )
+
+ second_lib_test(name = "second_lib_test", target_under_test = ":second", target_compatible_with = NOT_WINDOWS)
+ bin_test(name = "bin_test", target_under_test = ":bin", target_compatible_with = NOT_WINDOWS)
+
+def _rmeta_is_propagated_through_custom_rule_test_impl(ctx):
+ env = analysistest.begin(ctx)
+ tut = analysistest.target_under_test(env)
+
+ # This is the metadata-generating action. It should depend on metadata for the library and, if generate_metadata is set
+ # also depend on metadata for 'wrapper'.
+ rust_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0]
+
+ metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rmeta")]
+ rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib")]
+
+ seen_wrapper_metadata = False
+ seen_to_wrap_metadata = False
+ for mi in metadata_inputs:
+ if "libwrapper" in mi.path:
+ seen_wrapper_metadata = True
+ if "libto_wrap" in mi.path:
+ seen_to_wrap_metadata = True
+
+ seen_wrapper_rlib = False
+ seen_to_wrap_rlib = False
+ for ri in rlib_inputs:
+ if "libwrapper" in ri.path:
+ seen_wrapper_rlib = True
+ if "libto_wrap" in ri.path:
+ seen_to_wrap_rlib = True
+
+ if ctx.attr.generate_metadata:
+ asserts.true(env, seen_wrapper_metadata, "expected dependency on metadata for 'wrapper' but not found")
+ asserts.false(env, seen_wrapper_rlib, "expected no dependency on object for 'wrapper' but it was found")
+ else:
+ asserts.true(env, seen_wrapper_rlib, "expected dependency on object for 'wrapper' but not found")
+ asserts.false(env, seen_wrapper_metadata, "expected no dependency on metadata for 'wrapper' but it was found")
+
+ asserts.true(env, seen_to_wrap_metadata, "expected dependency on metadata for 'to_wrap' but not found")
+ asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found")
+
+ return analysistest.end(env)
+
+def _rmeta_is_used_when_building_custom_rule_test_impl(ctx):
+ env = analysistest.begin(ctx)
+ tut = analysistest.target_under_test(env)
+
+ # This is the custom rule invocation of rustc.
+ rust_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0]
+
+ # We want to check that the action depends on metadata, regardless of ctx.attr.generate_metadata
+ seen_to_wrap_rlib = False
+ seen_to_wrap_rmeta = False
+ for act in rust_action.inputs.to_list():
+ if "libto_wrap" in act.path and act.path.endswith(".rlib"):
+ seen_to_wrap_rlib = True
+ elif "libto_wrap" in act.path and act.path.endswith(".rmeta"):
+ seen_to_wrap_rmeta = True
+
+ asserts.true(env, seen_to_wrap_rmeta, "expected dependency on metadata for 'to_wrap' but not found")
+ asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found")
+
+ return analysistest.end(env)
+
+rmeta_is_propagated_through_custom_rule_test = analysistest.make(_rmeta_is_propagated_through_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING)
+rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, config_settings = ENABLE_PIPELINING)
+
+def _custom_rule_test(generate_metadata, suffix):
+ rust_library(
+ name = "to_wrap" + suffix,
+ crate_name = "to_wrap",
+ srcs = ["custom_rule_test/to_wrap.rs"],
+ edition = "2021",
+ )
+ wrap(
+ name = "wrapper" + suffix,
+ crate_name = "wrapper",
+ target = ":to_wrap" + suffix,
+ generate_metadata = generate_metadata,
+ )
+ rust_library(
+ name = "uses_wrapper" + suffix,
+ srcs = ["custom_rule_test/uses_wrapper.rs"],
+ deps = [":wrapper" + suffix],
+ edition = "2021",
+ )
+
+ rmeta_is_propagated_through_custom_rule_test(
+ name = "rmeta_is_propagated_through_custom_rule_test" + suffix,
+ generate_metadata = generate_metadata,
+ target_compatible_with = NOT_WINDOWS,
+ target_under_test = ":uses_wrapper" + suffix,
+ )
+
+ rmeta_is_used_when_building_custom_rule_test(
+ name = "rmeta_is_used_when_building_custom_rule_test" + suffix,
+ target_compatible_with = NOT_WINDOWS,
+ target_under_test = ":wrapper" + suffix,
+ )
+
+def pipelined_compilation_test_suite(name):
+ """Entry-point macro called from the BUILD file.
+
+ Args:
+ name: Name of the macro.
+ """
+ _pipelined_compilation_test()
+ _custom_rule_test(generate_metadata = True, suffix = "_with_metadata")
+ _custom_rule_test(generate_metadata = False, suffix = "_without_metadata")
+
+ native.test_suite(
+ name = name,
+ tests = [
+ ":bin_test",
+ ":second_lib_test",
+ ":rmeta_is_propagated_through_custom_rule_test_with_metadata",
+ ":rmeta_is_propagated_through_custom_rule_test_without_metadata",
+ ":rmeta_is_used_when_building_custom_rule_test_with_metadata",
+ ":rmeta_is_used_when_building_custom_rule_test_without_metadata",
+ ],
+ )
diff --git a/test/unit/pipelined_compilation/second.rs b/test/unit/pipelined_compilation/second.rs
new file mode 100644
index 0000000..b42e0b4
--- /dev/null
+++ b/test/unit/pipelined_compilation/second.rs
@@ -0,0 +1,7 @@
+use first::first_fun;
+use my_macro::noop;
+
+#[noop]
+pub fn fun() {
+ println!("{}", first_fun())
+}
diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl
new file mode 100644
index 0000000..11b8480
--- /dev/null
+++ b/test/unit/pipelined_compilation/wrap.bzl
@@ -0,0 +1,105 @@
+"""A custom rule that wraps a crate called to_wrap."""
+
+# buildifier: disable=bzl-visibility
+load("//rust/private:common.bzl", "rust_common")
+
+# buildifier: disable=bzl-visibility
+load("//rust/private:providers.bzl", "BuildInfo", "CrateInfo", "DepInfo", "DepVariantInfo")
+
+# buildifier: disable=bzl-visibility
+load("//rust/private:rustc.bzl", "rustc_compile_action")
+
+def _wrap_impl(ctx):
+ rs_file = ctx.actions.declare_file(ctx.label.name + "_wrapped.rs")
+ crate_name = ctx.attr.crate_name if ctx.attr.crate_name else ctx.label.name
+ ctx.actions.run_shell(
+ outputs = [rs_file],
+ command = """cat <<EOF > {}
+// crate_name: {}
+use to_wrap::to_wrap;
+
+pub fn wrap() {{
+ to_wrap();
+}}
+EOF
+""".format(rs_file.path, crate_name),
+ mnemonic = "WriteWrapperRsFile",
+ )
+
+ toolchain = ctx.toolchains[Label("//rust:toolchain")]
+
+ # Determine unique hash for this rlib
+ output_hash = repr(hash(rs_file.path))
+ crate_type = "rlib"
+
+ rust_lib_name = "{prefix}{name}-{lib_hash}{extension}".format(
+ prefix = "lib",
+ name = crate_name,
+ lib_hash = output_hash,
+ extension = ".rlib",
+ )
+ rust_metadata_name = "{prefix}{name}-{lib_hash}{extension}".format(
+ prefix = "lib",
+ name = crate_name,
+ lib_hash = output_hash,
+ extension = ".rmeta",
+ )
+
+ tgt = ctx.attr.target
+ deps = [DepVariantInfo(
+ crate_info = tgt[CrateInfo] if CrateInfo in tgt else None,
+ dep_info = tgt[DepInfo] if DepInfo in tgt else None,
+ build_info = tgt[BuildInfo] if BuildInfo in tgt else None,
+ cc_info = tgt[CcInfo] if CcInfo in tgt else None,
+ )]
+
+ rust_lib = ctx.actions.declare_file(rust_lib_name)
+ rust_metadata = None
+ if ctx.attr.generate_metadata:
+ rust_metadata = ctx.actions.declare_file(rust_metadata_name)
+ return rustc_compile_action(
+ ctx = ctx,
+ attr = ctx.attr,
+ toolchain = toolchain,
+ crate_info = rust_common.create_crate_info(
+ name = crate_name,
+ type = crate_type,
+ root = rs_file,
+ srcs = depset([rs_file]),
+ deps = depset(deps),
+ proc_macro_deps = depset([]),
+ aliases = {},
+ output = rust_lib,
+ metadata = rust_metadata,
+ owner = ctx.label,
+ edition = "2018",
+ compile_data = depset([]),
+ rustc_env = {},
+ is_test = False,
+ ),
+ output_hash = output_hash,
+ )
+
+wrap = rule(
+ implementation = _wrap_impl,
+ attrs = {
+ "crate_name": attr.string(),
+ "generate_metadata": attr.bool(default = False),
+ "target": attr.label(),
+ "_cc_toolchain": attr.label(
+ default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
+ ),
+ "_error_format": attr.label(
+ default = Label("//:error_format"),
+ ),
+ "_process_wrapper": attr.label(
+ default = Label("//util/process_wrapper"),
+ executable = True,
+ allow_single_file = True,
+ cfg = "exec",
+ ),
+ },
+ toolchains = ["@rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type"],
+ incompatible_use_toolchain_transition = True,
+ fragments = ["cpp"],
+)
diff --git a/test/unit/proc_macro/leaks_deps/lib/a.rs b/test/unit/proc_macro/leaks_deps/lib/a.rs
new file mode 100644
index 0000000..7d1a54e
--- /dev/null
+++ b/test/unit/proc_macro/leaks_deps/lib/a.rs
@@ -0,0 +1,5 @@
+use my_macro::greet;
+
+pub fn use_macro() -> &'static str {
+ greet!()
+}
diff --git a/test/unit/proc_macro/leaks_deps/lib/b.rs b/test/unit/proc_macro/leaks_deps/lib/b.rs
new file mode 100644
index 0000000..8fd0518
--- /dev/null
+++ b/test/unit/proc_macro/leaks_deps/lib/b.rs
@@ -0,0 +1,3 @@
+pub fn hello() -> &'static str {
+ "hello"
+}
diff --git a/test/unit/proc_macro/leaks_deps/lib/my_macro.rs b/test/unit/proc_macro/leaks_deps/lib/my_macro.rs
new file mode 100644
index 0000000..f8b6f2a
--- /dev/null
+++ b/test/unit/proc_macro/leaks_deps/lib/my_macro.rs
@@ -0,0 +1,7 @@
+use b::hello;
+use proc_macro::{Literal, TokenStream, TokenTree};
+
+#[proc_macro]
+pub fn greet(_item: TokenStream) -> TokenStream {
+ TokenTree::Literal(Literal::string(hello())).into()
+}
diff --git a/test/unit/proc_macro/leaks_deps/proc_macro_does_not_leak_deps.bzl b/test/unit/proc_macro/leaks_deps/proc_macro_does_not_leak_deps.bzl
index 40c2e48..db86506 100644
--- a/test/unit/proc_macro/leaks_deps/proc_macro_does_not_leak_deps.bzl
+++ b/test/unit/proc_macro/leaks_deps/proc_macro_does_not_leak_deps.bzl
@@ -1,7 +1,7 @@
"""Unittest to verify proc-macro targets"""
load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts")
-load("//rust:defs.bzl", "rust_proc_macro", "rust_test")
+load("//rust:defs.bzl", "rust_library", "rust_proc_macro", "rust_test")
def _proc_macro_does_not_leak_deps_impl(ctx):
env = analysistest.begin(ctx)
@@ -30,8 +30,6 @@
return analysistest.end(env)
-proc_macro_does_not_leak_deps_test = analysistest.make(_proc_macro_does_not_leak_deps_impl)
-
def _proc_macro_does_not_leak_deps_test():
rust_proc_macro(
name = "proc_macro_definition",
@@ -68,6 +66,70 @@
target_under_test = ":deps_not_leaked",
)
+proc_macro_does_not_leak_deps_test = analysistest.make(_proc_macro_does_not_leak_deps_impl)
+
+# Tests that a lib_a -> proc_macro -> lib_b does not propagate lib_b to the inputs of lib_a
+def _proc_macro_does_not_leak_lib_deps_impl(ctx):
+ env = analysistest.begin(ctx)
+ actions = analysistest.target_under_test(env).actions
+ rustc_actions = []
+ for action in actions:
+ if action.mnemonic == "Rustc" or action.mnemonic == "RustcMetadata":
+ rustc_actions.append(action)
+
+ # We should have a RustcMetadata and a Rustc action.
+ asserts.true(env, len(rustc_actions) == 2, "expected 2 actions, got %d" % len(rustc_actions))
+
+ for rustc_action in rustc_actions:
+ # lib :a has a dependency on :my_macro via a rust_proc_macro target.
+ # lib :b (which is a dependency of :my_macro) should not appear in the inputs of :a
+ b_inputs = [i for i in rustc_action.inputs.to_list() if "libb" in i.path]
+ b_args = [arg for arg in rustc_action.argv if "libb" in arg]
+
+ asserts.equals(env, 0, len(b_inputs))
+ asserts.equals(env, 0, len(b_args))
+
+ return analysistest.end(env)
+
+def _proc_macro_does_not_leak_lib_deps_test():
+ rust_library(
+ name = "b",
+ srcs = ["leaks_deps/lib/b.rs"],
+ edition = "2018",
+ )
+
+ rust_proc_macro(
+ name = "my_macro",
+ srcs = ["leaks_deps/lib/my_macro.rs"],
+ edition = "2018",
+ deps = [
+ ":b",
+ ],
+ )
+
+ rust_library(
+ name = "a",
+ srcs = ["leaks_deps/lib/a.rs"],
+ edition = "2018",
+ proc_macro_deps = [
+ ":my_macro",
+ ],
+ )
+
+ NOT_WINDOWS = select({
+ "@platforms//os:linux": [],
+ "@platforms//os:macos": [],
+ "//conditions:default": ["@platforms//:incompatible"],
+ })
+
+ proc_macro_does_not_leak_lib_deps_test(
+ name = "proc_macro_does_not_leak_lib_deps_test",
+ target_under_test = ":a",
+ target_compatible_with = NOT_WINDOWS,
+ )
+
+proc_macro_does_not_leak_lib_deps_test = analysistest.make(_proc_macro_does_not_leak_lib_deps_impl, config_settings = {"@//rust/settings:pipelined_compilation": True})
+
def proc_macro_does_not_leak_deps_test_suite(name):
"""Entry-point macro called from the BUILD file.
@@ -75,10 +137,12 @@
name: Name of the macro.
"""
_proc_macro_does_not_leak_deps_test()
+ _proc_macro_does_not_leak_lib_deps_test()
native.test_suite(
name = name,
tests = [
":proc_macro_does_not_leak_deps_test",
+ ":proc_macro_does_not_leak_lib_deps_test",
],
)
diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs
index a90bf4c..6d985b3 100644
--- a/util/process_wrapper/main.rs
+++ b/util/process_wrapper/main.rs
@@ -55,19 +55,6 @@
Ok(v) => v,
};
- let stderr: Box<dyn io::Write + Send> = if let Some(stderr_file) = opts.stderr_file {
- Box::new(
- OpenOptions::new()
- .create(true)
- .truncate(true)
- .write(true)
- .open(stderr_file)
- .expect("process wrapper error: unable to open stderr file"),
- )
- } else {
- Box::new(io::stderr())
- };
-
let mut child = Command::new(opts.executable)
.args(opts.child_arguments)
.env_clear()
@@ -87,25 +74,45 @@
.spawn()
.expect("process wrapper error: failed to spawn child process");
- let child_stderr = Box::new(child.stderr.take().unwrap());
+ let mut stderr: Box<dyn io::Write> = if let Some(stderr_file) = opts.stderr_file {
+ Box::new(
+ OpenOptions::new()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(stderr_file)
+ .expect("process wrapper error: unable to open stderr file"),
+ )
+ } else {
+ Box::new(io::stderr())
+ };
+
+ let mut child_stderr = child.stderr.take().unwrap();
let mut was_killed = false;
- let result = if !opts.rustc_quit_on_rmeta {
- // Process output normally by forwarding stderr
- process_output(child_stderr, stderr, LineOutput::Message)
- } else {
- let format = opts.rustc_output_format;
- let mut kill = false;
- let result = process_output(child_stderr, stderr, |line| {
- rustc::stop_on_rmeta_completion(line, format, &mut kill)
+ let result = if let Some(format) = opts.rustc_output_format {
+ let quit_on_rmeta = opts.rustc_quit_on_rmeta;
+ // Process json rustc output and kill the subprocess when we get a signal
+ // that we emitted a metadata file.
+ let mut me = false;
+ let metadata_emitted = &mut me;
+ let result = process_output(&mut child_stderr, stderr.as_mut(), move |line| {
+ if quit_on_rmeta {
+ rustc::stop_on_rmeta_completion(line, format, metadata_emitted)
+ } else {
+ rustc::process_json(line, format)
+ }
});
- if kill {
+ if me {
// If recv returns Ok(), a signal was sent in this channel so we should terminate the child process.
// We can safely ignore the Result from kill() as we don't care if the process already terminated.
let _ = child.kill();
was_killed = true;
}
result
+ } else {
+ // Process output normally by forwarding stderr
+ process_output(&mut child_stderr, stderr.as_mut(), LineOutput::Message)
};
result.expect("process wrapper error: failed to process stderr");
diff --git a/util/process_wrapper/options.rs b/util/process_wrapper/options.rs
index fdd60b4..869b5c3 100644
--- a/util/process_wrapper/options.rs
+++ b/util/process_wrapper/options.rs
@@ -44,7 +44,7 @@
pub(crate) rustc_quit_on_rmeta: bool,
// If rustc_quit_on_rmeta is set to true, this controls the
// output format of rustc messages.
- pub(crate) rustc_output_format: rustc::ErrorFormat,
+ pub(crate) rustc_output_format: Option<rustc::ErrorFormat>,
}
pub(crate) fn options() -> Result<Options, OptionError> {
@@ -173,8 +173,7 @@
v
))),
})
- .transpose()?
- .unwrap_or_default();
+ .transpose()?;
// Prepare the environment variables, unifying those read from files with the ones
// of the current process.
diff --git a/util/process_wrapper/output.rs b/util/process_wrapper/output.rs
index 049090c..84d61d9 100644
--- a/util/process_wrapper/output.rs
+++ b/util/process_wrapper/output.rs
@@ -31,8 +31,8 @@
/// Depending on the result of process_line, the modified message may be written
/// to write_end.
pub(crate) fn process_output<F>(
- read_end: Box<dyn Read>,
- write_end: Box<dyn Write>,
+ read_end: &mut dyn Read,
+ write_end: &mut dyn Write,
mut process_line: F,
) -> io::Result<()>
where
diff --git a/util/process_wrapper/rustc.rs b/util/process_wrapper/rustc.rs
index e566727..ca79680 100644
--- a/util/process_wrapper/rustc.rs
+++ b/util/process_wrapper/rustc.rs
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+use std::convert::{TryFrom, TryInto};
+
use tinyjson::JsonValue;
use crate::output::LineOutput;
@@ -40,9 +42,46 @@
}
}
-/// stop_on_rmeta_completion takes an output line from rustc configured with
+#[derive(Debug)]
+enum RustcMessage {
+ Emit(String),
+ Message(String),
+}
+
+impl TryFrom<JsonValue> for RustcMessage {
+ type Error = ();
+ fn try_from(val: JsonValue) -> Result<Self, Self::Error> {
+ if let Some(emit) = get_key(&val, "emit") {
+ return Ok(Self::Emit(emit));
+ }
+ if let Some(rendered) = get_key(&val, "rendered") {
+ return Ok(Self::Message(rendered));
+ }
+ Err(())
+ }
+}
+
+/// process_rustc_json takes an output line from rustc configured with
/// --error-format=json, parses the json and returns the appropriate output
-/// according to the original --error-format supplied to rustc.
+/// according to the original --error-format supplied.
+/// Only messages are returned, emits are ignored.
+pub(crate) fn process_json(line: String, error_format: ErrorFormat) -> LineOutput {
+ let parsed: JsonValue = line
+ .parse()
+ .expect("process wrapper error: expected json messages in pipeline mode");
+ match parsed.try_into() {
+ Ok(RustcMessage::Message(msg)) => match error_format {
+ // If the output should be json, we just forward the messages as-is
+ // using `line`.
+ ErrorFormat::Json => LineOutput::Message(line),
+ // Otherwise we return the rendered field.
+ _ => LineOutput::Message(msg),
+ },
+ _ => LineOutput::Skip,
+ }
+}
+
+/// stop_on_rmeta_completion parses the json output of rustc in the same way process_rustc_json does.
/// In addition, it will signal to stop when metadata is emitted
/// so the compiler can be terminated.
/// This is used to implement pipelining in rules_rust, please see
@@ -55,24 +94,19 @@
let parsed: JsonValue = line
.parse()
.expect("process wrapper error: expected json messages in pipeline mode");
- if let Some(emit) = get_key(&parsed, "emit") {
- // We don't want to print emit messages.
- // If the emit messages is "metadata" we can signal the process to quit
- return if emit == "metadata" {
+
+ match parsed.try_into() {
+ Ok(RustcMessage::Emit(emit)) if emit == "metadata" => {
*kill = true;
LineOutput::Terminate
- } else {
- LineOutput::Skip
- };
- };
-
- match error_format {
- // If the output should be json, we just forward the messages as-is
- ErrorFormat::Json => LineOutput::Message(line),
- // Otherwise we extract the "rendered" attribute.
- // If we don't find it we skip the line.
- _ => get_key(&parsed, "rendered")
- .map(LineOutput::Message)
- .unwrap_or(LineOutput::Skip),
+ }
+ Ok(RustcMessage::Message(msg)) => match error_format {
+ // If the output should be json, we just forward the messages as-is
+ // using `line`.
+ ErrorFormat::Json => LineOutput::Message(line),
+ // Otherwise we return the rendered field.
+ _ => LineOutput::Message(msg),
+ },
+ _ => LineOutput::Skip,
}
}
