Add test coverage support (#1324)
* WIP coverage support
* Make it work
* Regenerate documentation
* Gate env vars on coverage being enabled
* Add coverage targets
* Allow 2 actions when generating coverage
Co-authored-by: Krasimir Georgiev <krasimir@google.com>
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index 0e45f50..b278618 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -25,6 +25,7 @@
ubuntu2004:
build_targets: *default_linux_targets
test_targets: *default_linux_targets
+ coverage_targets: *default_linux_targets
rbe_ubuntu1604:
shell_commands:
- sed -i 's/^# load("@bazelci_rules/load("@bazelci_rules/' WORKSPACE.bazel
@@ -39,6 +40,7 @@
macos:
build_targets: *default_macos_targets
test_targets: *default_macos_targets
+ coverage_targets: *default_macos_targets
build_flags: *aspects_flags
windows:
build_flags:
diff --git a/docs/flatten.md b/docs/flatten.md
index f77e2da..563c69b 100644
--- a/docs/flatten.md
+++ b/docs/flatten.md
@@ -1062,9 +1062,9 @@
<pre>
rust_toolchain(<a href="#rust_toolchain-name">name</a>, <a href="#rust_toolchain-allocator_library">allocator_library</a>, <a href="#rust_toolchain-binary_ext">binary_ext</a>, <a href="#rust_toolchain-cargo">cargo</a>, <a href="#rust_toolchain-clippy_driver">clippy_driver</a>, <a href="#rust_toolchain-debug_info">debug_info</a>,
- <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>,
- <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>,
- <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
+ <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_cov">llvm_cov</a>, <a href="#rust_toolchain-llvm_profdata">llvm_profdata</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>,
+ <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>,
+ <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
</pre>
Declares a Rust toolchain for use.
@@ -1123,6 +1123,8 @@
| <a id="rust_toolchain-dylib_ext"></a>dylib_ext | The extension for dynamic libraries created from rustc. | String | required | |
| <a id="rust_toolchain-env"></a>env | Environment variables to set in actions. | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
| <a id="rust_toolchain-exec_triple"></a>exec_triple | The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations | String | required | |
+| <a id="rust_toolchain-llvm_cov"></a>llvm_cov | The location of the <code>llvm-cov</code> binary. Can be a direct source or a filegroup containing one item. | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| <a id="rust_toolchain-llvm_profdata"></a>llvm_profdata | The location of the <code>llvm-profdata</code> binary. Can be a direct source or a filegroup containing one item. | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
| <a id="rust_toolchain-llvm_tools"></a>llvm_tools | LLVM tools that are shipped with the Rust toolchain. | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
| <a id="rust_toolchain-opt_level"></a>opt_level | Rustc optimization levels. | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} |
| <a id="rust_toolchain-os"></a>os | The operating system for the current toolchain | String | required | |
diff --git a/docs/rust_repositories.md b/docs/rust_repositories.md
index fa077fc..bdce440 100644
--- a/docs/rust_repositories.md
+++ b/docs/rust_repositories.md
@@ -35,9 +35,9 @@
<pre>
rust_toolchain(<a href="#rust_toolchain-name">name</a>, <a href="#rust_toolchain-allocator_library">allocator_library</a>, <a href="#rust_toolchain-binary_ext">binary_ext</a>, <a href="#rust_toolchain-cargo">cargo</a>, <a href="#rust_toolchain-clippy_driver">clippy_driver</a>, <a href="#rust_toolchain-debug_info">debug_info</a>,
- <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>,
- <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>,
- <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
+ <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_cov">llvm_cov</a>, <a href="#rust_toolchain-llvm_profdata">llvm_profdata</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>,
+ <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>,
+ <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
</pre>
Declares a Rust toolchain for use.
@@ -96,6 +96,8 @@
| <a id="rust_toolchain-dylib_ext"></a>dylib_ext | The extension for dynamic libraries created from rustc. | String | required | |
| <a id="rust_toolchain-env"></a>env | Environment variables to set in actions. | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
| <a id="rust_toolchain-exec_triple"></a>exec_triple | The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations | String | required | |
+| <a id="rust_toolchain-llvm_cov"></a>llvm_cov | The location of the <code>llvm-cov</code> binary. Can be a direct source or a filegroup containing one item. | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| <a id="rust_toolchain-llvm_profdata"></a>llvm_profdata | The location of the <code>llvm-profdata</code> binary. Can be a direct source or a filegroup containing one item. | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
| <a id="rust_toolchain-llvm_tools"></a>llvm_tools | LLVM tools that are shipped with the Rust toolchain. | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
| <a id="rust_toolchain-opt_level"></a>opt_level | Rustc optimization levels. | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} |
| <a id="rust_toolchain-os"></a>os | The operating system for the current toolchain | String | required | |
diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl
index 96dd4e7..7fa2317 100644
--- a/rust/private/repository_utils.bzl
+++ b/rust/private/repository_utils.bzl
@@ -137,6 +137,35 @@
system = triple_to_system(target_triple)
return _build_file_for_clippy_template.format(binary_ext = system_to_binary_ext(system))
+_build_file_for_llvm_tools = """\
+filegroup(
+ name = "llvm_cov_bin",
+ srcs = ["lib/rustlib/{target_triple}/bin/llvm-cov{binary_ext}"],
+ visibility = ["//visibility:public"],
+)
+
+filegroup(
+ name = "llvm_profdata_bin",
+ srcs = ["lib/rustlib/{target_triple}/bin/llvm-profdata{binary_ext}"],
+ visibility = ["//visibility:public"],
+)
+"""
+
+def BUILD_for_llvm_tools(target_triple):
+ """Emits a BUILD file the llvm-tools binaries.
+
+ Args:
+ target_triple (str): The triple of the target platform
+
+ Returns:
+ str: The contents of a BUILD file
+ """
+ system = triple_to_system(target_triple)
+ return _build_file_for_llvm_tools.format(
+ binary_ext = system_to_binary_ext(system),
+ target_triple = target_triple,
+ )
+
_build_file_for_stdlib_template = """\
load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup")
@@ -189,6 +218,8 @@
rustfmt = {rustfmt_label},
cargo = "@{workspace_name}//:cargo",
clippy_driver = "@{workspace_name}//:clippy_driver_bin",
+ llvm_cov = {llvm_cov_label},
+ llvm_profdata = {llvm_profdata_label},
rustc_lib = "@{workspace_name}//:rustc_lib",
rustc_srcs = {rustc_srcs},
binary_ext = "{binary_ext}",
@@ -211,6 +242,7 @@
include_rustc_srcs,
default_edition,
include_rustfmt,
+ include_llvm_tools,
stdlib_linkflags = None):
"""Emits a toolchain declaration to match an existing compiler and stdlib.
@@ -222,6 +254,7 @@
include_rustc_srcs (bool, optional): Whether to download rustc's src code. This is required in order to use rust-analyzer support. Defaults to False.
default_edition (str): Default Rust edition.
include_rustfmt (bool): Whether rustfmt is present in the toolchain.
+ include_llvm_tools (bool): Whether llvm-tools are present in the toolchain.
stdlib_linkflags (list, optional): Overriden flags needed for linking to rust
stdlib, akin to BAZEL_LINKLIBS. Defaults to
None.
@@ -240,6 +273,11 @@
rustfmt_label = "None"
if include_rustfmt:
rustfmt_label = "\"@{workspace_name}//:rustfmt_bin\"".format(workspace_name = workspace_name)
+ llvm_cov_label = "None"
+ llvm_profdata_label = "None"
+ if include_llvm_tools:
+ llvm_cov_label = "\"@{workspace_name}//:llvm_cov_bin\"".format(workspace_name = workspace_name)
+ llvm_profdata_label = "\"@{workspace_name}//:llvm_profdata_bin\"".format(workspace_name = workspace_name)
return _build_file_for_rust_toolchain_template.format(
toolchain_name = name,
@@ -254,6 +292,8 @@
exec_triple = exec_triple,
target_triple = target_triple,
rustfmt_label = rustfmt_label,
+ llvm_cov_label = llvm_cov_label,
+ llvm_profdata_label = llvm_profdata_label,
)
_build_file_for_toolchain_template = """\
@@ -369,12 +409,13 @@
)""",
)
-def load_rust_stdlib(ctx, target_triple):
+def load_rust_stdlib(ctx, target_triple, include_llvm_tools):
"""Loads a rust standard library and yields corresponding BUILD for it
Args:
ctx (repository_ctx): A repository_ctx.
target_triple (str): The rust-style target triple of the tool
+ include_llvm_tools (bool): Whether to include LLVM tools in the toolchain
Returns:
str: The BUILD file contents for this stdlib, and a toolchain decl to match
@@ -408,6 +449,7 @@
workspace_name = ctx.attr.name,
default_edition = ctx.attr.edition,
include_rustfmt = not (not ctx.attr.rustfmt_version),
+ include_llvm_tools = include_llvm_tools,
)
return stdlib_build_file + toolchain_build_file
@@ -449,6 +491,8 @@
version = ctx.attr.version,
)
+ return BUILD_for_llvm_tools(target_triple)
+
def check_version_valid(version, iso_date, param_prefix = ""):
"""Verifies that the provided rust version and iso_date make sense.
diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl
index 81a86b1..3299c5c 100644
--- a/rust/private/rust.bzl
+++ b/rust/private/rust.bzl
@@ -466,6 +466,9 @@
getattr(ctx.attr, "env", {}),
data,
)
+ if ctx.configuration.coverage_enabled:
+ env["RUST_LLVM_COV"] = toolchain.llvm_cov.path
+ env["RUST_LLVM_PROFDATA"] = toolchain.llvm_profdata.path
providers.append(testing.TestEnvironment(env))
return providers
@@ -649,6 +652,11 @@
"_cc_toolchain": attr.label(
default = "@bazel_tools//tools/cpp:current_cc_toolchain",
),
+ "_collect_cc_coverage": attr.label(
+ default = "//util:collect_coverage",
+ executable = True,
+ cfg = "exec",
+ ),
"_error_format": attr.label(default = "//:error_format"),
"_extra_exec_rustc_flags": attr.label(default = "//:extra_exec_rustc_flags"),
"_extra_rustc_flags": attr.label(default = "//:extra_rustc_flags"),
diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl
index d48adfb..29c5e4c 100644
--- a/rust/private/rustc.bzl
+++ b/rust/private/rustc.bzl
@@ -805,6 +805,9 @@
rustc_flags.add("--extern")
rustc_flags.add("proc_macro")
+ if ctx.configuration.coverage_enabled:
+ rustc_flags.add("--codegen=instrument-coverage")
+
# Make bin crate data deps available to tests.
for data in getattr(attr, "data", []):
if rust_common.crate_info in data:
@@ -993,8 +996,12 @@
),
)
+ coverage_runfiles = []
+ if toolchain.llvm_cov and ctx.configuration.coverage_enabled and crate_info.is_test:
+ coverage_runfiles = [toolchain.llvm_cov, toolchain.llvm_profdata]
+
runfiles = ctx.runfiles(
- files = getattr(ctx.files, "data", []),
+ files = getattr(ctx.files, "data", []) + coverage_runfiles,
collect_data = True,
)
@@ -1009,6 +1016,12 @@
runfiles = runfiles,
executable = crate_info.output if crate_info.type == "bin" or crate_info.is_test or out_binary else None,
),
+ coverage_common.instrumented_files_info(
+ ctx,
+ dependency_attributes = ["deps", "crate"],
+ extensions = ["rs"],
+ source_attributes = ["srcs"],
+ ),
]
if crate_info.type in ["staticlib", "cdylib"]:
diff --git a/rust/repositories.bzl b/rust/repositories.bzl
index e532954..2880db4 100644
--- a/rust/repositories.bzl
+++ b/rust/repositories.bzl
@@ -173,11 +173,12 @@
build_components.append(load_rustfmt(ctx))
# Rust 1.45.0 and nightly builds after 2020-05-22 need the llvm-tools gzip to get the libLLVM dylib
- if ctx.attr.version >= "1.45.0" or (ctx.attr.version == "nightly" and ctx.attr.iso_date > "2020-05-22"):
- load_llvm_tools(ctx, ctx.attr.exec_triple)
+ include_llvm_tools = ctx.attr.version >= "1.45.0" or (ctx.attr.version == "nightly" and ctx.attr.iso_date > "2020-05-22")
+ if include_llvm_tools:
+ build_components.append(load_llvm_tools(ctx, ctx.attr.exec_triple))
for target_triple in [ctx.attr.exec_triple] + ctx.attr.extra_target_triples:
- build_components.append(load_rust_stdlib(ctx, target_triple))
+ build_components.append(load_rust_stdlib(ctx, target_triple, include_llvm_tools))
# extra_target_triples contains targets such as wasm, which don't have rustc_dev components
if ctx.attr.dev_components and target_triple not in ctx.attr.extra_target_triples:
diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl
index 90dea6e..b7a72ef 100644
--- a/rust/toolchain.bzl
+++ b/rust/toolchain.bzl
@@ -495,6 +495,8 @@
env = ctx.attr.env,
exec_triple = ctx.attr.exec_triple,
libstd_and_allocator_ccinfo = _make_libstd_and_allocator_ccinfo(ctx, rust_std, ctx.attr.allocator_library),
+ llvm_cov = ctx.file.llvm_cov,
+ llvm_profdata = ctx.file.llvm_profdata,
make_variables = platform_common.TemplateVariableInfo(make_variables),
os = ctx.attr.os,
rust_doc = sysroot.rustdoc,
@@ -569,6 +571,16 @@
),
mandatory = True,
),
+ "llvm_cov": attr.label(
+ doc = "The location of the `llvm-cov` binary. Can be a direct source or a filegroup containing one item.",
+ allow_single_file = True,
+ cfg = "exec",
+ ),
+ "llvm_profdata": attr.label(
+ doc = "The location of the `llvm-profdata` binary. Can be a direct source or a filegroup containing one item.",
+ allow_single_file = True,
+ cfg = "exec",
+ ),
"llvm_tools": attr.label(
doc = "LLVM tools that are shipped with the Rust toolchain.",
allow_files = True,
diff --git a/test/unit/native_deps/native_deps_test.bzl b/test/unit/native_deps/native_deps_test.bzl
index 202cf09..d066c80 100644
--- a/test/unit/native_deps/native_deps_test.bzl
+++ b/test/unit/native_deps/native_deps_test.bzl
@@ -48,7 +48,10 @@
def _proc_macro_has_native_libs_test_impl(ctx):
env = analysistest.begin(ctx)
tut = analysistest.target_under_test(env)
- asserts.equals(env, 1, len(tut.actions))
+ if ctx.configuration.coverage_enabled:
+ asserts.equals(env, 2, len(tut.actions))
+ else:
+ asserts.equals(env, 1, len(tut.actions))
action = tut.actions[0]
assert_argv_contains_prefix_suffix(env, action, "-Lnative=", "/native_deps")
assert_argv_contains(env, action, "--crate-type=proc-macro")
diff --git a/util/BUILD.bazel b/util/BUILD.bazel
index 6f6453a..83dbc1e 100644
--- a/util/BUILD.bazel
+++ b/util/BUILD.bazel
@@ -16,3 +16,9 @@
],
visibility = ["//:__subpackages__"],
)
+
+filegroup(
+ name = "collect_coverage",
+ srcs = ["collect_coverage.sh"],
+ visibility = ["//visibility:public"],
+)
diff --git a/util/collect_coverage.sh b/util/collect_coverage.sh
new file mode 100755
index 0000000..06fdd2a
--- /dev/null
+++ b/util/collect_coverage.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ -n "${VERBOSE_COVERAGE:-}" ]]; then
+ set -x
+fi
+
+readonly profdata_file=$COVERAGE_DIR/coverage.profdata
+
+"$RUNFILES_DIR/$TEST_WORKSPACE/$RUST_LLVM_PROFDATA" \
+ merge \
+ --sparse "$COVERAGE_DIR"/*.profraw \
+ -output "$profdata_file"
+
+"$RUNFILES_DIR/$TEST_WORKSPACE/$RUST_LLVM_COV" \
+ export \
+ -format=lcov \
+ -instr-profile "$profdata_file" \
+ -ignore-filename-regex='.*external/.+' \
+ -ignore-filename-regex='/tmp/.+' \
+ -path-equivalence="$ROOT",. \
+ "$RUNFILES_DIR/$TEST_WORKSPACE/$TEST_BINARY" \
+ | sed 's#/proc/self/cwd/##' > "$COVERAGE_OUTPUT_FILE"
