| #!/usr/bin/env fuchsia-vendored-python | |
| # | |
| # Copyright 2020 The Fuchsia Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| import argparse | |
| import collections | |
| import copy | |
| import datetime | |
| import enum | |
| import functools | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import sys | |
| import textwrap | |
| ROOT_PATH = os.path.abspath(__file__ + "/../..") | |
| sys.path += [os.path.join(ROOT_PATH, "third_party", "pytoml")] | |
| import pytoml as toml | |
| CARGO_PACKAGE_CONTENTS = """\ | |
| # Copyright %(year)s The Fuchsia Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| # source GN: %(target)s" | |
| cargo-features = ["per-package-target"] | |
| [package] | |
| name = "%(package_name)s" | |
| version = "%(version)s" | |
| license = "BSD-3-Clause" | |
| authors = ["rust-fuchsia@fuchsia.com"] | |
| description = "Rust crate for Fuchsia OS" | |
| repository = "https://fuchsia.googlesource.com" | |
| edition = "%(edition)s" | |
| %(default_target)s | |
| %(bin_or_lib)s | |
| %(is_proc_macro)s | |
| name = "%(crate_name)s" | |
| path = "%(source_root)s" | |
| """ | |
| CARGO_PACKAGE_NO_WORKSPACE = """\ | |
| [workspace] | |
| # empty workspace table excludes this crate from thinking it should be in a workspace | |
| """ | |
| CARGO_PACKAGE_DEP = """\ | |
| [%(dep_type)s.%(crate_name)s] | |
| version = "%(version)s" | |
| path = "%(crate_path)s" | |
| """ | |
| class ToolchainType(enum.Enum): | |
| TARGET = 1 | |
| HOST = 2 | |
| def strip_toolchain(target): | |
| return re.search("[^(]*", target)[0] | |
| def extract_toolchain(target): | |
| """ Return the toolchain part of the provided label, or None if it doesn't have one.""" | |
| if "(" not in target: | |
| # target has no toolchain specified | |
| return None | |
| substr = target[target.find("(") + 1:] | |
| if not target.endswith(")"): | |
| raise ValueError("target %s missing closing `)`") | |
| return substr[:-1] # remove closing `)` | |
| def version_from_toolchain(toolchain): | |
| """ Return a version to use to allow host and target crates to coexist. """ | |
| version = "0.0.1" | |
| if toolchain != None and toolchain.startswith("//build/toolchain:host_"): | |
| version = "0.0.2" | |
| return version | |
| def classify_toolchain(toolchain): | |
| if toolchain != None and toolchain.startswith("//build/toolchain:host_"): | |
| return ToolchainType.HOST | |
| else: | |
| return ToolchainType.TARGET | |
| def lookup_gn_pkg_name(project, target, *, for_workspace): | |
| if for_workspace: | |
| return mangle_label(target) | |
| metadata = project.targets[target] | |
| return metadata["output_name"] | |
| def rebase_gn_path(root_path, location, directory=False): | |
| assert location[0:2] == "//" | |
| # remove the prefix // | |
| path = location[2:] | |
| target = os.path.dirname(path) if directory else path | |
| return os.path.join(root_path, target) | |
| def mangle_label(label): | |
| assert label[0:2] == "//" | |
| # remove the prefix // | |
| label = label[2:] | |
| result = [] | |
| for c in label: | |
| if c == "-": | |
| result.append("--") | |
| elif c == "_": | |
| result.append("__") | |
| elif c == "/": | |
| result.append("_-_") | |
| elif c == ":": | |
| result.append("-_-") | |
| elif c == ".": | |
| result.append("_--") | |
| elif c == "(" or c == ")": | |
| result.append("-__") | |
| else: | |
| result.append(c) | |
| return "".join(result) | |
| class FeatureSpec(object): | |
| def __init__(self, features, default_features): | |
| self.features = features | |
| self.default_features = default_features | |
| class Project(object): | |
| def __init__(self, project_json): | |
| self.targets = project_json | |
| self.patches = None | |
| self.third_party_features = {} | |
| @functools.cached_property | |
| def rust_targets(self): | |
| return { | |
| target: meta | |
| for target, meta in self.targets.items() | |
| if "crate_root" in meta | |
| } | |
| @functools.cached_property | |
| def rust_targets_by_source_root(self): | |
| result = collections.defaultdict(list) | |
| for target, meta in self.rust_targets.items(): | |
| source_root = meta["crate_root"] | |
| result[source_root].append(target) | |
| return dict(result) | |
| @functools.cached_property | |
| def reachable_targets(self): | |
| result = set(["//:default"]) | |
| pending = ["//:default"] | |
| while pending: | |
| current = pending.pop() | |
| meta = self.targets[current] | |
| for dep_kind in ["deps", "public_deps", "data_deps"]: | |
| for dep in meta.get(dep_kind, []): | |
| if dep not in result: | |
| result.add(dep) | |
| pending.append(dep) | |
| return result | |
| def expand_source_set_or_group(self, target): | |
| """Returns a list of dependencies if the target is a source_set. | |
| Returns dependencies as a list of strings if the target is a | |
| source_set, or None otherwise. | |
| """ | |
| meta = self.targets[target] | |
| if meta["type"] in ("source_set", "group"): | |
| return meta["deps"] | |
| def find_test_targets(self, source_root): | |
| overlapping_targets = self.rust_targets_by_source_root.get( | |
| source_root, []) | |
| return [ | |
| t for t in overlapping_targets | |
| if "--test" in self.targets[t]["rustflags"] | |
| ] | |
| def write_toml_file( | |
| fout, metadata, project, target, lookup, root_path, root_build_dir, | |
| gn_cargo_dir, for_workspace, version): | |
| rust_crates_path = os.path.join(root_path, "third_party/rust_crates") | |
| editions = [ | |
| flag.split("=")[1] | |
| for flag in metadata["rustflags"] | |
| if flag.startswith("--edition=") | |
| ] | |
| edition = editions[0] if editions else "2015" | |
| if metadata["type"] in ["rust_library", "rust_proc_macro", | |
| "static_library"]: | |
| target_type = "[lib]" | |
| else: | |
| if "--test" in metadata["rustflags"]: | |
| target_type = "[[test]]" | |
| else: | |
| target_type = "[[bin]]" | |
| if metadata["type"] == "rust_proc_macro": | |
| is_proc_macro = "proc-macro = true" | |
| else: | |
| is_proc_macro = "" | |
| features = [] | |
| extra_configs = [] | |
| feature_pat = re.compile(r"--cfg=feature=\"(.*)\"$") | |
| cfg_pat = re.compile(r"--cfg=([^=]*)=(.*)$") | |
| for flag in metadata["rustflags"]: | |
| match = feature_pat.match(flag) | |
| if match: | |
| features.append(match.group(1)) | |
| elif match := cfg_pat.match(flag): | |
| if match.group(1) != "__rust_toolchain": | |
| extra_configs.append(f"{match.group(1)}={match.group(2)}") | |
| elif flag.startswith("--cfg="): | |
| extra_configs.append(flag[len("--cfg="):]) | |
| crate_type = "rlib" | |
| package_name = lookup_gn_pkg_name( | |
| project, target, for_workspace=for_workspace) | |
| default_target = "" | |
| if classify_toolchain(extract_toolchain(target)) == ToolchainType.TARGET: | |
| default_target = 'default-target = "x86_64-fuchsia"' | |
| fout.write( | |
| CARGO_PACKAGE_CONTENTS % { | |
| "target": target, | |
| "package_name": package_name, | |
| "crate_name": metadata["crate_name"], | |
| "version": version, | |
| "year": datetime.datetime.now().year, | |
| "bin_or_lib": target_type, | |
| "is_proc_macro": is_proc_macro, | |
| "lib_crate_type": crate_type, | |
| "edition": edition, | |
| "source_root": rebase_gn_path(root_path, metadata["crate_root"]), | |
| "rust_crates_path": rust_crates_path, | |
| "default_target": default_target, | |
| }) | |
| env_vars = metadata.get("rustenv", []) | |
| if extra_configs or env_vars: | |
| with open(os.path.join(gn_cargo_dir, str(lookup[target]), "build.rs"), | |
| "w") as buildfile: | |
| template = textwrap.dedent( | |
| """\ | |
| //! build script for {target} | |
| fn main() {{ | |
| // build script does not read any files | |
| println!("cargo:rerun-if-changed=build.rs"); | |
| {body} | |
| {env_vars} | |
| }} | |
| """) | |
| body = "\n".join( | |
| f'println!(r#"cargo:rustc-cfg={cfg}"#);' | |
| for cfg in extra_configs) | |
| env_vars = "\n".join( | |
| f'println!("cargo:rustc-env={env}");' for env in env_vars) | |
| buildfile.write( | |
| template.format(target=target, body=body, env_vars=env_vars)) | |
| extra_test_deps = set() | |
| if target_type in {"[lib]", "[[bin]]"}: | |
| test_targets = project.find_test_targets(metadata["crate_root"]) | |
| # hack to filter to just matching toolchains: | |
| test_targets = [ | |
| t for t in test_targets if t.split("(")[1:] == target.split("(")[1:] | |
| ] | |
| test_deps = set() | |
| for test_target in test_targets: | |
| test_deps.update(project.targets[test_target]["deps"]) | |
| unreachable_test_deps = sorted( | |
| [dep for dep in test_deps if dep not in project.reachable_targets]) | |
| if unreachable_test_deps: | |
| fout.write( | |
| "# Note: disabling tests because test deps are not included in the build: %s\n" | |
| % unreachable_test_deps) | |
| fout.write("test = false\n") | |
| elif not test_targets: | |
| fout.write( | |
| "# Note: disabling tests because no test target was found with the same source root\n" | |
| ) | |
| fout.write("test = false\n") | |
| else: | |
| fout.write( | |
| "# Note: using extra deps from discovered test target(s): %s\n" | |
| % test_targets) | |
| extra_test_deps = sorted(test_deps - set(metadata["deps"])) | |
| if not for_workspace: | |
| fout.write(CARGO_PACKAGE_NO_WORKSPACE) | |
| if not for_workspace: | |
| # In a workspace, patches are ignored, so we skip emitting all the patch lines to cut down on warning spam | |
| fout.write("\n[patch.crates-io]\n") | |
| for patch in project.patches: | |
| path = project.patches[patch]["path"] | |
| fout.write( | |
| "%s = { path = \"%s/%s\"" % (patch, rust_crates_path, path)) | |
| if package := project.patches[patch].get("package"): | |
| fout.write(", package = \"%s\"" % (package,)) | |
| fout.write(" }\n") | |
| fout.write("\n") | |
| def expand_and_deduplicate(deps, visited=None): | |
| if visited is None: | |
| visited = set() | |
| for dep in deps: | |
| if dep in visited: | |
| continue | |
| visited.add(dep) | |
| expanded = project.expand_source_set_or_group(dep) | |
| if expanded: | |
| for exp in expand_and_deduplicate(expanded, visited): | |
| yield exp | |
| else: | |
| yield dep | |
| # collect all dependencies | |
| deps = list(expand_and_deduplicate(metadata["deps"])) | |
| dep_crate_names = set() | |
| def write_deps(deps, dep_type): | |
| while deps: | |
| dep = deps.pop() | |
| # If a dependency points to a source set or group, expand it into a list | |
| # of its deps, and append them to the deps list. Finally, continue | |
| # to the next item, since a source set itself is not considered a | |
| # dependency for our purposes. | |
| expanded_deps = project.expand_source_set_or_group(dep) | |
| if expanded_deps: | |
| deps.extend(expanded_deps) | |
| continue | |
| # this is a third-party dependency | |
| # TODO remove this when all things use GN. temporary hack? | |
| if "third_party/rust_crates:" in dep: | |
| match = re.search("rust_crates:([\w-]*)", dep) | |
| crate_name, version = str(match.group(1)).rsplit("-v", 1) | |
| dep_crate_names.add(crate_name) | |
| version = version.replace("_", ".") | |
| feature_spec = project.third_party_features.get(crate_name) | |
| fout.write("[%s.\"%s\"]\n" % (dep_type, crate_name)) | |
| fout.write("version = \"%s\"\n" % version) | |
| if feature_spec: | |
| fout.write( | |
| "features = %s\n" % json.dumps(feature_spec.features)) | |
| if feature_spec.default_features is False: | |
| fout.write("default-features = false\n") | |
| if crate_name in features: | |
| # Make the dependency optional if there is a feature with | |
| # the same name. Later, we'll make sure to list the feature | |
| # in the default feature list so that we do actually include | |
| # the dependency. | |
| fout.write("optional = true\n") | |
| # this is a in-tree rust target | |
| elif "crate_name" in project.targets[dep]: | |
| toolchain = extract_toolchain(dep) | |
| version = version_from_toolchain(toolchain) | |
| crate_name = lookup_gn_pkg_name( | |
| project, dep, for_workspace=for_workspace) | |
| dep_dir = os.path.join(gn_cargo_dir, str(lookup[dep])) | |
| fout.write( | |
| CARGO_PACKAGE_DEP % { | |
| "dep_type": dep_type, | |
| "crate_path": dep_dir, | |
| "crate_name": crate_name, | |
| "version": version, | |
| }) | |
| write_deps(deps, "dependencies") | |
| write_deps(extra_test_deps, "dev-dependencies") | |
| if features: | |
| fout.write("\n[features]\n") | |
| # Filter 'default' feature out to avoid generating a duplicated entry. | |
| features = [x for x in features if x != "default"] | |
| fout.write("default = %s\n" % json.dumps(features)) | |
| for feature in features: | |
| # Filter features that are also dependencies | |
| # https://users.rust-lang.org/t/features-and-dependencies-cannot-have-the-same-name/47746/2 | |
| if feature not in dep_crate_names: | |
| fout.write("%s = []\n" % feature) | |
| def main(): | |
| # TODO(tmandry): Remove all hardcoded paths and replace with args. | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--root_build_dir", required=True) | |
| parser.add_argument("--fuchsia_dir", required=True) | |
| parser.add_argument("json_path") | |
| args = parser.parse_args() | |
| json_path = args.json_path | |
| project = None | |
| try: | |
| with open(json_path, "r") as json_file: | |
| project = json.loads(json_file.read()) | |
| except IOError: | |
| print("Failed to generate Cargo.toml files") | |
| print("No project.json in the root of your out directory!") | |
| print("Run gn with the --ide=json flag set") | |
| # returns 0 so that CQ doesn't fail if this isn't set properly | |
| return 0 | |
| project = Project(project) | |
| root_path = os.path.abspath(args.fuchsia_dir) | |
| root_build_dir = os.path.abspath(args.root_build_dir) | |
| rust_crates_path = os.path.join(root_path, "third_party/rust_crates") | |
| # this will be removed eventually? | |
| with open(rust_crates_path + "/Cargo.toml", "r") as f: | |
| cargo_toml = toml.load(f) | |
| project.patches = cargo_toml["patch"]["crates-io"] | |
| # Map from crate name to FeatureSpec. We don't include the version because we don't directly | |
| # depend on more than one version of the same crate. | |
| def collect_features(deps): | |
| for dep, info in deps.items(): | |
| if isinstance(info, str): | |
| continue | |
| project.third_party_features[dep] = FeatureSpec( | |
| info.get("features", []), info.get("default-features", True)) | |
| collect_features(cargo_toml["dependencies"]) | |
| for target_info in cargo_toml["target"].values(): | |
| collect_features(target_info.get("dependencies", {})) | |
| lookup = {} | |
| for target in project.rust_targets: | |
| # hash is the GN target name without the prefixed // | |
| lookup[target] = hashlib.sha1(target[2:].encode("utf-8")).hexdigest() | |
| # remove the priorly generated rust crates | |
| gn_cargo_dir = os.path.join(root_build_dir, "cargo") | |
| shutil.rmtree(gn_cargo_dir, ignore_errors=True) | |
| os.makedirs(gn_cargo_dir) | |
| # Write a stamp file with a predictable name so the build system knows the | |
| # step ran successfully. | |
| with open(os.path.join(gn_cargo_dir, "generate_cargo.stamp"), "w") as f: | |
| f.truncate() | |
| # a dict of "toolchain label" to list of Cargo.toml files in it | |
| # special case: the key None means the default toolchain | |
| workspace_dirs_by_toolchain = collections.defaultdict(list) | |
| for target in project.rust_targets: | |
| toolchain = extract_toolchain(target) | |
| version = version_from_toolchain(toolchain) | |
| cargo_toml_dir = os.path.join(gn_cargo_dir, str(lookup[target])) | |
| try: | |
| os.makedirs(cargo_toml_dir) | |
| except OSError: | |
| print("Failed to create directory for Cargo: %s" % cargo_toml_dir) | |
| for_workspace_cargo_toml_dir = os.path.join( | |
| gn_cargo_dir, "for_workspace", str(lookup[target])) | |
| try: | |
| os.makedirs(for_workspace_cargo_toml_dir) | |
| except OSError: | |
| print( | |
| "Failed to create directory for Cargo: %s" % | |
| for_workspace_cargo_toml_dir) | |
| metadata = project.targets[target] | |
| with open(os.path.join(cargo_toml_dir, "Cargo.toml"), "w") as fout: | |
| write_toml_file( | |
| fout, | |
| metadata, | |
| project, | |
| target, | |
| lookup, | |
| root_path, | |
| root_build_dir, | |
| gn_cargo_dir, | |
| for_workspace=False, | |
| version=version) | |
| if (not target.startswith("//third_party/rust_crates:") | |
| ) and target in project.reachable_targets: | |
| workspace_dirs_by_toolchain[toolchain].append( | |
| ( | |
| target, | |
| os.path.relpath(for_workspace_cargo_toml_dir, root_path))) | |
| with open(os.path.join(for_workspace_cargo_toml_dir, "Cargo.toml"), | |
| "w") as fout: | |
| write_toml_file( | |
| fout, | |
| metadata, | |
| project, | |
| target, | |
| lookup, | |
| root_path, | |
| root_build_dir, | |
| os.path.join(gn_cargo_dir, "for_workspace"), | |
| for_workspace=True, | |
| version=version) | |
| # TODO: refactor into separate function | |
| for toolchain, workspace_dirs in workspace_dirs_by_toolchain.items(): | |
| subdir = os.path.join(gn_cargo_dir, "for_workspace") | |
| if toolchain: | |
| # Strip off the leading "//" from the toolchain label so we don't | |
| # accidentally use it as an absolute path. | |
| path_safe_toolchain = toolchain.lstrip("/") | |
| subdir = os.path.join(subdir, "toolchain", path_safe_toolchain) | |
| else: | |
| # the workspace for the default toolchain (None in the dict) just | |
| # lives in for_workspace directly. | |
| pass | |
| try: | |
| os.makedirs(subdir, exist_ok=True) | |
| except OSError: | |
| print("Failed to create directory for Cargo: %s" % subdir) | |
| with open(os.path.join(subdir, "Cargo_for_fuchsia_dir.toml"), | |
| "w") as fout: | |
| fout.write("[workspace]\nmembers = [\n") | |
| for target, dir in workspace_dirs: | |
| fout.write(" # %s\n" % target) | |
| fout.write(" %s,\n" % json.dumps(dir)) | |
| fout.write("]\n") | |
| fout.write('exclude = ["third_party/rust_crates",]\n') | |
| fout.write("\n[patch.crates-io]\n") | |
| for patch in project.patches: | |
| path = project.patches[patch]["path"] | |
| fout.write( | |
| "%s = { path = %s" % ( | |
| patch, | |
| json.dumps( | |
| os.path.join("third_party/rust_crates", path)))) | |
| if package := project.patches[patch].get("package"): | |
| fout.write(", package = \"%s\"" % (package,)) | |
| fout.write(" }\n") | |
| fout.write("\n") | |
| rust_targets = sorted( | |
| [ | |
| { | |
| "label": t, | |
| "crate_name": project.targets[t]["crate_name"], | |
| "type": project.targets[t]["type"], | |
| "cargo_manifest_dir": lookup[t], | |
| } for t in project.rust_targets if t in project.reachable_targets | |
| ], | |
| key=lambda t: t["label"]) | |
| # Returns a single rust target per "base" label (not including toolchain), | |
| # either for fuchsia toolchains or host toolchains. This is used for rustdoc | |
| # where we only want to document each crate once for fuchsia and once for host. | |
| def rustdoc_targets(host): | |
| cur = "" | |
| result = [] | |
| for t in rust_targets: | |
| if t["type"] == "executable": | |
| continue | |
| if meta := project.rust_targets[t["label"]].get("metadata"): | |
| if disable := meta.get("disable_rustdoc"): | |
| if disable == [True]: | |
| continue | |
| l = t["label"].replace(".actual", "") | |
| base = l.split("(")[0] | |
| is_host = "(//build/toolchain:host" in l | |
| if host == is_host: | |
| if base == cur: | |
| continue | |
| cur = base | |
| result.append(copy.deepcopy(t)) | |
| result[-1]["label"] = l | |
| return result | |
| def dump_json(obj, filename): | |
| with open(os.path.join(gn_cargo_dir, filename), "w") as f: | |
| json.dump(obj, f) | |
| dump_json(rust_targets, "rust_targets.json") | |
| dump_json(rustdoc_targets(host=False), "rustdoc_targets.json") | |
| dump_json(rustdoc_targets(host=True), "rustdoc_host_targets.json") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) |