crate_universe/extension.bzl

"""Module extension for generating third-party crates for use in bazel."""

load("@bazel_features//:features.bzl", "bazel_features")
load("@bazel_skylib//lib:structs.bzl", "structs")
load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//crate_universe:defs.bzl", _crate_universe_crate = "crate")
load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest")
load("//crate_universe/private:generate_utils.bzl", "CARGO_BAZEL_GENERATOR_SHA256", "CARGO_BAZEL_GENERATOR_URL", "GENERATOR_ENV_VARS", "render_config")
load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS")
load("//crate_universe/private/module_extensions:cargo_bazel_bootstrap.bzl", "get_cargo_bazel_runner")
load("//rust/platform:triple.bzl", "get_host_triple")

# A list of labels which may be relative (and if so, is within the repo the rule is generated in).
#
# If I were to write ":foo", with attr.label_list, it would evaluate to
# "@@//:foo". However, for a tag such as deps, ":foo" should refer to
# "@@rules_rust~crates~<crate>//:foo".
_relative_label_list = attr.string_list

_OPT_BOOL_VALUES = {
    "auto": None,
    "off": False,
    "on": True,
}

def optional_bool(doc):
    return attr.string(
        doc = doc,
        values = _OPT_BOOL_VALUES.keys(),
        default = "auto",
    )

def _get_or_insert(d, key, value):
    if key not in d:
        d[key] = value
    return d[key]

def _generate_repo_impl(repo_ctx):
    for path, contents in repo_ctx.attr.contents.items():
        repo_ctx.file(path, contents)

_generate_repo = repository_rule(
    implementation = _generate_repo_impl,
    attrs = dict(
        contents = attr.string_dict(mandatory = True),
    ),
)

def _annotations_for_repo(module_annotations, repo_specific_annotations):
    """Merges the set of global annotations with the repo-specific ones

    Args:
        module_annotations (dict): The annotation tags that apply to all repos, keyed by crate.
        repo_specific_annotations (dict): The annotation tags that apply to only this repo, keyed by crate.
    """

    if not repo_specific_annotations:
        return module_annotations

    annotations = dict(module_annotations)
    for crate, values in repo_specific_annotations.items():
        _get_or_insert(annotations, crate, []).extend(values)
    return annotations

def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, cargo_lockfile = None, manifests = {}, packages = {}):
    """Generates repositories for the transitive closure of crates defined by manifests and packages.

    Args:
        module_ctx (module_ctx): The module context object.
        cargo_bazel (function): A function that can be called to execute cargo_bazel.
        cfg (object): The module tag from `from_cargo` or `from_specs`
        annotations (dict): The set of annotation tag classes that apply to this closure, keyed by crate name.
        cargo_lockfile (path): Path to Cargo.lock, if we have one. This is optional for `from_specs` closures.
        manifests (dict): The set of Cargo.toml manifests that apply to this closure, if any, keyed by path.
        packages (dict): The set of extra cargo crate tags that apply to this closure, if any, keyed by package name.
    """

    tag_path = module_ctx.path(cfg.name)

    rendering_config = json.decode(render_config(
        regen_command = "Run 'cargo update [--workspace]'",
    ))
    config_file = tag_path.get_child("config.json")
    module_ctx.file(
        config_file,
        executable = False,
        content = generate_config_file(
            module_ctx,
            mode = "remote",
            annotations = annotations,
            generate_build_scripts = cfg.generate_build_scripts,
            supported_platform_triples = cfg.supported_platform_triples,
            generate_target_compatible_with = True,
            repository_name = cfg.name,
            output_pkg = cfg.name,
            workspace_name = cfg.name,
            generate_binaries = cfg.generate_binaries,
            render_config = rendering_config,
            repository_ctx = module_ctx,
        ),
    )

    splicing_manifest = tag_path.get_child("splicing_manifest.json")
    module_ctx.file(
        splicing_manifest,
        executable = False,
        content = generate_splicing_manifest(
            packages = packages,
            splicing_config = "",
            cargo_config = cfg.cargo_config,
            manifests = manifests,
            manifest_to_path = module_ctx.path,
        ),
    )

    splicing_output_dir = tag_path.get_child("splicing-output")
    splice_args = [
        "splice",
        "--output-dir",
        splicing_output_dir,
        "--config",
        config_file,
        "--splicing-manifest",
        splicing_manifest,
    ]
    if cargo_lockfile:
        splice_args.extend([
            "--cargo-lockfile",
            cargo_lockfile,
        ])
    cargo_bazel(splice_args)

    # Create a lockfile, since we need to parse it to generate spoke
    # repos.
    lockfile_path = tag_path.get_child("lockfile.json")
    module_ctx.file(lockfile_path, "")

    cargo_bazel([
        "generate",
        "--cargo-lockfile",
        cargo_lockfile or splicing_output_dir.get_child("Cargo.lock"),
        "--config",
        config_file,
        "--splicing-manifest",
        splicing_manifest,
        "--repository-dir",
        tag_path,
        "--metadata",
        splicing_output_dir.get_child("metadata.json"),
        "--repin",
        "--lockfile",
        lockfile_path,
    ])

    crates_dir = tag_path.get_child(cfg.name)
    _generate_repo(
        name = cfg.name,
        contents = {
            "BUILD.bazel": module_ctx.read(crates_dir.get_child("BUILD.bazel")),
            "defs.bzl": module_ctx.read(crates_dir.get_child("defs.bzl")),
        },
    )

    contents = json.decode(module_ctx.read(lockfile_path))

    for crate in contents["crates"].values():
        repo = crate["repository"]
        if repo == None:
            continue
        name = crate["name"]
        version = crate["version"]

        # "+" isn't valid in a repo name.
        crate_repo_name = "{repo_name}__{name}-{version}".format(
            repo_name = cfg.name,
            name = name,
            version = version.replace("+", "-"),
        )

        build_file_content = module_ctx.read(crates_dir.get_child("BUILD.%s-%s.bazel" % (name, version)))
        if "Http" in repo:
            # Replicates functionality in repo_http.j2.
            repo = repo["Http"]
            http_archive(
                name = crate_repo_name,
                patch_args = repo.get("patch_args", None),
                patch_tool = repo.get("patch_tool", None),
                patches = repo.get("patches", None),
                remote_patch_strip = 1,
                sha256 = repo.get("sha256", None),
                type = "tar.gz",
                urls = [repo["url"]],
                strip_prefix = "%s-%s" % (crate["name"], crate["version"]),
                build_file_content = build_file_content,
            )
        elif "Git" in repo:
            # Replicates functionality in repo_git.j2
            repo = repo["Git"]
            kwargs = {}
            for k, v in repo["commitish"].items():
                if k == "Rev":
                    kwargs["commit"] = v
                else:
                    kwargs[k.lower()] = v
            new_git_repository(
                name = crate_repo_name,
                init_submodules = True,
                patch_args = repo.get("patch_args", None),
                patch_tool = repo.get("patch_tool", None),
                patches = repo.get("patches", None),
                shallow_since = repo.get("shallow_since", None),
                remote = repo["remote"],
                build_file_content = build_file_content,
                strip_prefix = repo.get("strip_prefix", None),
                **kwargs
            )
        else:
            fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo))

def _package_to_json(p):
    # Avoid adding unspecified properties.
    # If we add them as empty strings, cargo-bazel will be unhappy.
    return json.encode({
        k: v
        for k, v in structs.to_dict(p).items()
        if v or k == "default_features"
    })

def _get_generator(module_ctx):
    """Query Network Resources to local a `cargo-bazel` binary.  

    Based off get_generator in crates_universe/private/generate_utils.bzl

    Args:
        module_ctx (module_ctx):  The rules context object

    Returns:
        tuple(path, dict) The path to a 'cargo-bazel' binary. The pairing (dict)
            may be `None` if there is not need to update the attribute
    """
    host_triple = get_host_triple(module_ctx)
    use_environ = False
    for var in GENERATOR_ENV_VARS:
        if var in module_ctx.os.environ:
            use_environ = True

    if use_environ:
        generator_sha256 = module_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_SHA256)
        generator_url = module_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_URL)
    elif len(CARGO_BAZEL_URLS) == 0:
        return module_ctx.path(Label("@cargo_bazel_bootstrap//:cargo-bazel"))
    else:
        generator_sha256 = CARGO_BAZEL_SHA256S.get(host_triple.str)
        generator_url = CARGO_BAZEL_URLS.get(host_triple.str)

    if not generator_url:
        fail((
            "No generator URL was found either in the `CARGO_BAZEL_GENERATOR_URL` " +
            "environment variable or for the `{}` triple in the `generator_urls` attribute"
        ).format(host_triple.str))

    output = module_ctx.path("cargo-bazel.exe" if "win" in module_ctx.os.name else "cargo-bazel")

    # Download the file into place
    download_kwargs = {
        "executable": True,
        "output": output,
        "url": generator_url,
    }

    if generator_sha256:
        download_kwargs.update({"sha256": generator_sha256})

    module_ctx.download(**download_kwargs)
    return output

def _crate_impl(module_ctx):
    cargo_bazel_output = _get_generator(module_ctx)
    cargo_bazel = get_cargo_bazel_runner(module_ctx, cargo_bazel_output)

    all_repos = []
    reproducible = True

    for mod in module_ctx.modules:
        module_annotations = {}
        repo_specific_annotations = {}
        for annotation_tag in mod.tags.annotation:
            annotation_dict = structs.to_dict(annotation_tag)
            repositories = annotation_dict.pop("repositories")
            crate = annotation_dict.pop("crate")

            # The crate.annotation function can take in either a list or a bool.
            # For the tag-based method, because it has type safety, we have to
            # split it into two parameters.
            if annotation_dict.pop("gen_all_binaries"):
                annotation_dict["gen_binaries"] = True
            annotation_dict["gen_build_script"] = _OPT_BOOL_VALUES[annotation_dict["gen_build_script"]]
            annotation = _crate_universe_crate.annotation(**{
                k: v
                for k, v in annotation_dict.items()
                # Tag classes can't take in None, but the function requires None
                # instead of the empty values in many cases.
                # https://github.com/bazelbuild/bazel/issues/20744
                if v != "" and v != [] and v != {}
            })
            if not repositories:
                _get_or_insert(module_annotations, crate, []).append(annotation)
            for repo in repositories:
                _get_or_insert(
                    _get_or_insert(repo_specific_annotations, repo, {}),
                    crate,
                    [],
                ).append(annotation)

        local_repos = []

        for cfg in mod.tags.from_cargo + mod.tags.from_specs:
            if cfg.name in local_repos:
                fail("Defined two crate universes with the same name in the same MODULE.bazel file. Use the name tag to give them different names.")
            elif cfg.name in all_repos:
                fail("Defined two crate universes with the same name in different MODULE.bazel files. Either give one a different name, or use use_extension(isolate=True)")
            all_repos.append(cfg.name)
            local_repos.append(cfg.name)

        for cfg in mod.tags.from_cargo:
            annotations = _annotations_for_repo(
                module_annotations,
                repo_specific_annotations.get(cfg.name),
            )

            cargo_lockfile = module_ctx.path(cfg.cargo_lockfile)
            manifests = {str(module_ctx.path(m)): str(m) for m in cfg.manifests}
            _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, cargo_lockfile = cargo_lockfile, manifests = manifests)

        for cfg in mod.tags.from_specs:
            # We don't have a Cargo.lock so the resolution can change.
            # We could maybe make this reproducible by using `-minimal-version` during resolution.
            # See https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#minimal-versions
            reproducible = False

            annotations = _annotations_for_repo(
                module_annotations,
                repo_specific_annotations.get(cfg.name),
            )

            packages = {p.package: _package_to_json(p) for p in mod.tags.spec}
            _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, packages = packages)

        for repo in repo_specific_annotations:
            if repo not in local_repos:
                fail("Annotation specified for repo %s, but the module defined repositories %s" % (repo, local_repos))

    metadata_kwargs = {}
    if bazel_features.external_deps.extension_metadata_has_reproducible:
        metadata_kwargs["reproducible"] = reproducible

    return module_ctx.extension_metadata(**metadata_kwargs)

_from_cargo = tag_class(
    doc = "Generates a repo @crates from a Cargo.toml / Cargo.lock pair",
    attrs = dict(
        name = attr.string(
            doc = "The name of the repo to generate",
            default = "crates",
        ),
        cargo_lockfile = CRATES_VENDOR_ATTRS["cargo_lockfile"],
        manifests = CRATES_VENDOR_ATTRS["manifests"],
        cargo_config = CRATES_VENDOR_ATTRS["cargo_config"],
        generate_binaries = CRATES_VENDOR_ATTRS["generate_binaries"],
        generate_build_scripts = CRATES_VENDOR_ATTRS["generate_build_scripts"],
        supported_platform_triples = CRATES_VENDOR_ATTRS["supported_platform_triples"],
    ),
)

# This should be kept in sync with crate_universe/private/crate.bzl.
_annotation = tag_class(
    attrs = dict(
        repositories = attr.string_list(
            doc = "A list of repository names specified from `crate.from_cargo(name=...)` that this annotation is applied to. Defaults to all repositories.",
            default = [],
        ),
        crate = attr.string(
            doc = "The name of the crate the annotation is applied to",
            mandatory = True,
        ),
        version = attr.string(
            doc = "The versions of the crate the annotation is applied to. Defaults to all versions.",
            default = "*",
        ),
        additive_build_file_content = attr.string(
            doc = "Extra contents to write to the bottom of generated BUILD files.",
        ),
        additive_build_file = attr.label(
            doc = "A file containing extra contents to write to the bottom of generated BUILD files.",
        ),
        alias_rule = attr.string(
            doc = "Alias rule to use instead of `native.alias()`.  Overrides [render_config](#render_config)'s 'default_alias_rule'.",
        ),
        build_script_data = _relative_label_list(
            doc = "A list of labels to add to a crate's `cargo_build_script::data` attribute.",
        ),
        build_script_tools = _relative_label_list(
            doc = "A list of labels to add to a crate's `cargo_build_script::tools` attribute.",
        ),
        build_script_data_glob = attr.string_list(
            doc = "A list of glob patterns to add to a crate's `cargo_build_script::data` attribute",
        ),
        build_script_deps = _relative_label_list(
            doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.",
        ),
        build_script_env = attr.string_dict(
            doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.",
        ),
        build_script_proc_macro_deps = _relative_label_list(
            doc = "A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute.",
        ),
        build_script_rundir = attr.string(
            doc = "An override for the build script's rundir attribute.",
        ),
        build_script_rustc_env = attr.string_dict(
            doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.",
        ),
        build_script_toolchains = attr.label_list(
            doc = "A list of labels to set on a crates's `cargo_build_script::toolchains` attribute.",
        ),
        compile_data = _relative_label_list(
            doc = "A list of labels to add to a crate's `rust_library::compile_data` attribute.",
        ),
        compile_data_glob = attr.string_list(
            doc = "A list of glob patterns to add to a crate's `rust_library::compile_data` attribute.",
        ),
        crate_features = attr.string_list(
            doc = "A list of strings to add to a crate's `rust_library::crate_features` attribute.",
        ),
        data = _relative_label_list(
            doc = "A list of labels to add to a crate's `rust_library::data` attribute.",
        ),
        data_glob = attr.string_list(
            doc = "A list of glob patterns to add to a crate's `rust_library::data` attribute.",
        ),
        deps = _relative_label_list(
            doc = "A list of labels to add to a crate's `rust_library::deps` attribute.",
        ),
        extra_aliased_targets = attr.string_dict(
            doc = "A list of targets to add to the generated aliases in the root crate_universe repository.",
        ),
        gen_binaries = attr.string_list(
            doc = "As a list, the subset of the crate's bins that should get `rust_binary` targets produced.",
        ),
        gen_all_binaries = attr.bool(
            doc = "If true, generates `rust_binary` targets for all of the crates bins",
        ),
        disable_pipelining = attr.bool(
            doc = "If True, disables pipelining for library targets for this crate.",
        ),
        gen_build_script = attr.string(
            doc = "An authorative flag to determine whether or not to produce `cargo_build_script` targets for the current crate. Supported values are 'on', 'off', and 'auto'.",
            values = _OPT_BOOL_VALUES.keys(),
            default = "auto",
        ),
        patch_args = attr.string_list(
            doc = "The `patch_args` attribute of a Bazel repository rule. See [http_archive.patch_args](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_args)",
        ),
        patch_tool = attr.string(
            doc = "The `patch_tool` attribute of a Bazel repository rule. See [http_archive.patch_tool](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_tool)",
        ),
        patches = attr.label_list(
            doc = "The `patches` attribute of a Bazel repository rule. See [http_archive.patches](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patches)",
        ),
        proc_macro_deps = _relative_label_list(
            doc = "A list of labels to add to a crate's `rust_library::proc_macro_deps` attribute.",
        ),
        rustc_env = attr.string_dict(
            doc = "Additional variables to set on a crate's `rust_library::rustc_env` attribute.",
        ),
        rustc_env_files = _relative_label_list(
            doc = "A list of labels to set on a crate's `rust_library::rustc_env_files` attribute.",
        ),
        rustc_flags = attr.string_list(
            doc = "A list of strings to set on a crate's `rust_library::rustc_flags` attribute.",
        ),
        shallow_since = attr.string(
            doc = "An optional timestamp used for crates originating from a git repository instead of a crate registry. This flag optimizes fetching the source code.",
        ),
    ),
)

_from_specs = tag_class(
    doc = "Generates a repo @crates from the defined `spec` tags",
    attrs = dict(
        name = attr.string(doc = "The name of the repo to generate", default = "crates"),
        cargo_config = CRATES_VENDOR_ATTRS["cargo_config"],
        generate_binaries = CRATES_VENDOR_ATTRS["generate_binaries"],
        generate_build_scripts = CRATES_VENDOR_ATTRS["generate_build_scripts"],
        supported_platform_triples = CRATES_VENDOR_ATTRS["supported_platform_triples"],
    ),
)

# This should be kept in sync with crate_universe/private/crate.bzl.
_spec = tag_class(
    attrs = dict(
        package = attr.string(
            doc = "The explicit name of the package.",
            mandatory = True,
        ),
        version = attr.string(
            doc = "The exact version of the crate. Cannot be used with `git`.",
        ),
        artifact = attr.string(
            doc = "Set to 'bin' to pull in a binary crate as an artifact dependency. Requires a nightly Cargo.",
        ),
        lib = attr.bool(
            doc = "If using `artifact = 'bin'`, additionally setting `lib = True` declares a dependency on both the package's library and binary, as opposed to just the binary.",
        ),
        default_features = attr.bool(
            doc = "Maps to the `default-features` flag.",
            default = True,
        ),
        features = attr.string_list(
            doc = "A list of features to use for the crate.",
        ),
        git = attr.string(
            doc = "The Git url to use for the crate. Cannot be used with `version`.",
        ),
        branch = attr.string(
            doc = "The git branch of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.",
        ),
        tag = attr.string(
            doc = "The git tag of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.",
        ),
        rev = attr.string(
            doc = "The git revision of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified.",
        ),
    ),
)

crate = module_extension(
    implementation = _crate_impl,
    tag_classes = dict(
        from_cargo = _from_cargo,
        annotation = _annotation,
        from_specs = _from_specs,
        spec = _spec,
    ),
)