recipes/contrib/firebase_docs.py - infra/recipes - Git at Google

 # Copyright 2018 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Recipe for generating docs for upload to Firebase."""

 from PB.recipes.fuchsia.contrib.firebase_docs import InputProperties

 DEPS = [
     "fuchsia/build",
     "fuchsia/buildbucket_util",
     "fuchsia/checkout",
     "fuchsia/git_checkout",
     "fuchsia/python3",
     "recipe_engine/cipd",
     "recipe_engine/context",
     "recipe_engine/file",
     "recipe_engine/json",
     "recipe_engine/path",
     "recipe_engine/properties",
     "recipe_engine/step",
 ]

 GENERATORS = ["rustdoc", "clangdoc"]

 REFERENCE_DOCS_REPOSITORY = "https://fuchsia.googlesource.com/reference-docs"
 API_DOCS_RESOURCES_REPOSITORY = "https://fuchsia.googlesource.com/api-docs-resources"

 PROPERTIES = InputProperties

 FIDLDOC_COMMIT_MESSAGE = "[fidldoc] Updating fidl reference docs"


 def RunSteps(api, props):
     if not props.generators:
         props.generators.extend(GENERATORS)

     cipd_dir = api.path.start_dir / "cipd"
     node_modules_dir = cipd_dir / "node_modules"
     with api.step.nest("ensure_packages"), api.context(infra_steps=True):
         pkgs = api.cipd.EnsureFile()
         pkgs.add_package("infra/nodejs/nodejs/${platform}", "latest")
         api.cipd.ensure(cipd_dir, pkgs)

         # firebase-tools expects to live in the node_modules subdir of where nodejs is installed.
         pkgs = api.cipd.EnsureFile()
         pkgs.add_package("infra/npm/firebase-tools", "latest")
         api.cipd.ensure(node_modules_dir, pkgs)

     with api.step.nest("checkout api docs"):
         resources_dir, _ = api.git_checkout(API_DOCS_RESOURCES_REPOSITORY)

     checkout = api.checkout.fuchsia_with_options(
         manifest=props.manifest, remote=props.remote
     )

     build_results = api.build.with_options(
         checkout=checkout, fint_params_path=props.fint_params_path
     )

     docs_dir = checkout.root_dir / "firebase"

     api.file.rmtree("remove old docs", docs_dir)
     api.file.copytree("copy resources", resources_dir, docs_dir)

     if "clangdoc" in props.generators:
         with api.step.nest("clangdoc"):
             gen_clang_doc(api, docs_dir, build_results)
     if "rustdoc" in props.generators:
         with api.step.nest("rustdoc"):
             gen_rustdoc(api, checkout.root_dir, docs_dir, build_results.build_dir)

     # Only deploy if running the default 'all' one, to avoid deploying partial docs to firebase.
     if not props.dry_run and props.output_name == "all":
         with api.context(cwd=docs_dir, env={"PATH": cipd_dir / "bin"}):
             api.step(
                 "firebase deploy",
                 [
                     node_modules_dir.joinpath(".bin", "firebase"),
                     "deploy",
                     "--only",
                     "hosting",
                     "--debug",
                 ],
             )


 def gen_rustdoc(api, checkout_root, docs_dir, build_dir):
     """Generate rust output.

     The rustdoc script runs on GN targets. We find the Rust GN targets by finding
     all targets that generate a Cargo.toml file, and use those.

     Args:
         docs_dir (Path): The output directory for documentation.
         build_dir (Path): The build directory.
     """

     default_targets = api.file.read_json(
         "read list of rust targets",
         build_dir / "cargo/rustdoc_targets.json",
         test_data=[
             {
                 "label": "//src/target:lib",
                 "type": "rust_library",
                 "cargo_manifest_dir": "289484b2c1b7271f566115d3b44610003ed64792",
             },
             {
                 "label": "//src/target:wasm_lib(//build/toolchain:unknown_wasm32)",
                 "type": "shared_library",
                 "cargo_manifest_dir": "7d824f223c1fd4f697010c18ad425b4ac1b83c50",
             },
         ],
     )
     host_targets = api.file.read_json(
         "read list of rust targets",
         build_dir / "cargo/rustdoc_host_targets.json",
         test_data=[
             {
                 "label": "//src/target:lib(//build/toolchain:host_x64)",
                 "type": "rust_library",
                 "cargo_manifest_dir": "96609c122ae77526a35b79752f30f7d89f6a0914",
             },
         ],
     )
     rust_targets = sorted(default_targets + host_targets, key=lambda t: t["label"])

     with api.step.nest("build docs"):
         for rust_target in rust_targets:
             target = rust_target["label"]
             cargo_toml_path = build_dir.joinpath(
                 "cargo", rust_target["cargo_manifest_dir"], "Cargo.toml"
             )
             # don't build docs for wasm targets, it's a very small proportion of our
             # crates and they break sometimes
             # TODO(fxbug.dev/97619): figure out if the toolchain is supposed to be
             # //build/toolchain:unknown_wasm32 or
             # //build/toolchain:unknown_wasm32-shared and add back the ")"
             if "(//build/toolchain:unknown_wasm32" in target:
                 continue

             try:
                 with api.context(
                     env={
                         "FUCHSIA_DIR": checkout_root,
                         "FUCHSIA_BUILD_DIR": build_dir,
                     },
                 ):
                     api.python3(
                         f"rustdoc {target}",
                         [
                             checkout_root.joinpath(
                                 "tools/devshell/contrib/lib/rust/rustdoc.py",
                             ),
                             cargo_toml_path,
                             "--no-deps",
                             "--out-dir",
                             build_dir,
                         ],
                     )
             except api.step.StepFailure:
                 pass

     cargo_dir = checkout_root.joinpath("out", "cargo_target")
     # Move the output to the docs directory.
     api.file.move(
         "move output to docs",
         cargo_dir.joinpath("x86_64-fuchsia", "doc"),
         docs_dir.joinpath("public", "rust"),
     )
     api.file.rmtree(
         "remove crate named 'host'",
         cargo_dir.joinpath("x86_64-fuchsia", "doc", "host"),
     )
     api.file.move(
         "move host output to docs",
         cargo_dir / "doc",
         docs_dir.joinpath("public", "rust", "host"),
     )


 def gen_clang_doc(api, docs_dir, build_results):
     """Generate clang-doc output.

     clang-doc runs on the translation units specified in the compilation database
     file. This file will be generated by gn_results after filtering unwanted
     directories or files. clang-doc will output documentation files directly in
     docs_dir.

     Args:
         docs_dir (Path): The output directory for documentation.
         build_results (FuchsiaBuildResults): Result of a fuchsia build.
     """

     # Assumes that the current builder is configured to generate compdbs.
     compdb = api.file.read_json(
         "read compdb",
         build_results.compdb_path,
         test_data=[
             {
                 "directory": "[START_DIR]/out/not-default",
                 "file": "../../foo.cpp",
                 "command": "clang++ foo.cpp",
             },
             {
                 "directory": "[START_DIR]/out/not-default",
                 "file": "../../third_party/foo.cpp",
                 "command": "clang++ third_party/foo.cpp",
             },
             {
                 "directory": "[START_DIR]/out/not-default",
                 "file": "../../out/not-default/foo.cpp",
                 "command": "clang++ foo.cpp",
             },
             {
                 "directory": "[START_DIR]/out/not-default",
                 "file": "/bin/ln",
                 "command": "/bin/ln -s foo bar",
             },
         ],
         include_log=False,
     )

     def keep_in_compdb(entry):
         # Filenames are relative to the build directory, and the build
         # directory is absolute.
         build_dir = api.path.abs_to_path(entry["directory"])
         abspath = api.path.realpath(api.path.join(build_dir, entry["file"]))
         try:
             path = api.path.abs_to_path(abspath)
         except ValueError:
             # This happens if `abspath` is not rooted in one of the path
             # module's known paths, which is the case for paths in `/bin`,
             # for example. That also implies that the path isn't in the
             # build directory, so skip it.
             return False

         if build_dir in path.parents:
             return False

         if "third_party" in entry["file"].split(api.path.sep):
             return False
         return True

     compdb_filtered = [entry for entry in compdb if keep_in_compdb(entry)]

     with api.context(cwd=build_results.checkout.root_dir):
         api.step(
             "run clang-doc",
             [
                 build_results.tool("clang-doc"),
                 "--output",
                 docs_dir.joinpath("public", "cpp"),
                 "--public",
                 "--format=html",
                 api.json.input(compdb_filtered),
             ],
         )


 def GenTests(api):
     def properties():
         return api.properties(
             remote="https://fuchsia.googlesource.com/integration",
             fint_params_path="specs/firebase-docs.textproto",
             manifest="flower",
             output_name="all",
         )

     yield (api.buildbucket_util.test("firebase_docs") + properties())
     yield (
         api.buildbucket_util.test("firebase_docs_failing_rustdoc")
         + properties()
         + api.step_data("rustdoc.build docs.rustdoc //src/target:lib", retcode=1)
     )
	# Copyright 2018 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Recipe for generating docs for upload to Firebase."""

	from PB.recipes.fuchsia.contrib.firebase_docs import InputProperties

	DEPS = [
	"fuchsia/build",
	"fuchsia/buildbucket_util",
	"fuchsia/checkout",
	"fuchsia/git_checkout",
	"fuchsia/python3",
	"recipe_engine/cipd",
	"recipe_engine/context",
	"recipe_engine/file",
	"recipe_engine/json",
	"recipe_engine/path",
	"recipe_engine/properties",
	"recipe_engine/step",
	]

	GENERATORS = ["rustdoc", "clangdoc"]

	REFERENCE_DOCS_REPOSITORY = "https://fuchsia.googlesource.com/reference-docs"
	API_DOCS_RESOURCES_REPOSITORY = "https://fuchsia.googlesource.com/api-docs-resources"

	PROPERTIES = InputProperties

	FIDLDOC_COMMIT_MESSAGE = "[fidldoc] Updating fidl reference docs"


	def RunSteps(api, props):
	if not props.generators:
	props.generators.extend(GENERATORS)

	cipd_dir = api.path.start_dir / "cipd"
	node_modules_dir = cipd_dir / "node_modules"
	with api.step.nest("ensure_packages"), api.context(infra_steps=True):
	pkgs = api.cipd.EnsureFile()
	pkgs.add_package("infra/nodejs/nodejs/${platform}", "latest")
	api.cipd.ensure(cipd_dir, pkgs)

	# firebase-tools expects to live in the node_modules subdir of where nodejs is installed.
	pkgs = api.cipd.EnsureFile()
	pkgs.add_package("infra/npm/firebase-tools", "latest")
	api.cipd.ensure(node_modules_dir, pkgs)

	with api.step.nest("checkout api docs"):
	resources_dir, _ = api.git_checkout(API_DOCS_RESOURCES_REPOSITORY)

	checkout = api.checkout.fuchsia_with_options(
	manifest=props.manifest, remote=props.remote
	)

	build_results = api.build.with_options(
	checkout=checkout, fint_params_path=props.fint_params_path
	)

	docs_dir = checkout.root_dir / "firebase"

	api.file.rmtree("remove old docs", docs_dir)
	api.file.copytree("copy resources", resources_dir, docs_dir)

	if "clangdoc" in props.generators:
	with api.step.nest("clangdoc"):
	gen_clang_doc(api, docs_dir, build_results)
	if "rustdoc" in props.generators:
	with api.step.nest("rustdoc"):
	gen_rustdoc(api, checkout.root_dir, docs_dir, build_results.build_dir)

	# Only deploy if running the default 'all' one, to avoid deploying partial docs to firebase.
	if not props.dry_run and props.output_name == "all":
	with api.context(cwd=docs_dir, env={"PATH": cipd_dir / "bin"}):
	api.step(
	"firebase deploy",
	[
	node_modules_dir.joinpath(".bin", "firebase"),
	"deploy",
	"--only",
	"hosting",
	"--debug",
	],
	)


	def gen_rustdoc(api, checkout_root, docs_dir, build_dir):
	"""Generate rust output.

	The rustdoc script runs on GN targets. We find the Rust GN targets by finding
	all targets that generate a Cargo.toml file, and use those.

	Args:
	docs_dir (Path): The output directory for documentation.
	build_dir (Path): The build directory.
	"""

	default_targets = api.file.read_json(
	"read list of rust targets",
	build_dir / "cargo/rustdoc_targets.json",
	test_data=[
	{
	"label": "//src/target:lib",
	"type": "rust_library",
	"cargo_manifest_dir": "289484b2c1b7271f566115d3b44610003ed64792",
	},
	{
	"label": "//src/target:wasm_lib(//build/toolchain:unknown_wasm32)",
	"type": "shared_library",
	"cargo_manifest_dir": "7d824f223c1fd4f697010c18ad425b4ac1b83c50",
	},
	],
	)
	host_targets = api.file.read_json(
	"read list of rust targets",
	build_dir / "cargo/rustdoc_host_targets.json",
	test_data=[
	{
	"label": "//src/target:lib(//build/toolchain:host_x64)",
	"type": "rust_library",
	"cargo_manifest_dir": "96609c122ae77526a35b79752f30f7d89f6a0914",
	},
	],
	)
	rust_targets = sorted(default_targets + host_targets, key=lambda t: t["label"])

	with api.step.nest("build docs"):
	for rust_target in rust_targets:
	target = rust_target["label"]
	cargo_toml_path = build_dir.joinpath(
	"cargo", rust_target["cargo_manifest_dir"], "Cargo.toml"
	)
	# don't build docs for wasm targets, it's a very small proportion of our
	# crates and they break sometimes
	# TODO(fxbug.dev/97619): figure out if the toolchain is supposed to be
	# //build/toolchain:unknown_wasm32 or
	# //build/toolchain:unknown_wasm32-shared and add back the ")"
	if "(//build/toolchain:unknown_wasm32" in target:
	continue

	try:
	with api.context(
	env={
	"FUCHSIA_DIR": checkout_root,
	"FUCHSIA_BUILD_DIR": build_dir,
	},
	):
	api.python3(
	f"rustdoc {target}",
	[
	checkout_root.joinpath(
	"tools/devshell/contrib/lib/rust/rustdoc.py",
	),
	cargo_toml_path,
	"--no-deps",
	"--out-dir",
	build_dir,
	],
	)
	except api.step.StepFailure:
	pass

	cargo_dir = checkout_root.joinpath("out", "cargo_target")
	# Move the output to the docs directory.
	api.file.move(
	"move output to docs",
	cargo_dir.joinpath("x86_64-fuchsia", "doc"),
	docs_dir.joinpath("public", "rust"),
	)
	api.file.rmtree(
	"remove crate named 'host'",
	cargo_dir.joinpath("x86_64-fuchsia", "doc", "host"),
	)
	api.file.move(
	"move host output to docs",
	cargo_dir / "doc",
	docs_dir.joinpath("public", "rust", "host"),
	)


	def gen_clang_doc(api, docs_dir, build_results):
	"""Generate clang-doc output.

	clang-doc runs on the translation units specified in the compilation database
	file. This file will be generated by gn_results after filtering unwanted
	directories or files. clang-doc will output documentation files directly in
	docs_dir.

	Args:
	docs_dir (Path): The output directory for documentation.
	build_results (FuchsiaBuildResults): Result of a fuchsia build.
	"""

	# Assumes that the current builder is configured to generate compdbs.
	compdb = api.file.read_json(
	"read compdb",
	build_results.compdb_path,
	test_data=[
	{
	"directory": "[START_DIR]/out/not-default",
	"file": "../../foo.cpp",
	"command": "clang++ foo.cpp",
	},
	{
	"directory": "[START_DIR]/out/not-default",
	"file": "../../third_party/foo.cpp",
	"command": "clang++ third_party/foo.cpp",
	},
	{
	"directory": "[START_DIR]/out/not-default",
	"file": "../../out/not-default/foo.cpp",
	"command": "clang++ foo.cpp",
	},
	{
	"directory": "[START_DIR]/out/not-default",
	"file": "/bin/ln",
	"command": "/bin/ln -s foo bar",
	},
	],
	include_log=False,
	)

	def keep_in_compdb(entry):
	# Filenames are relative to the build directory, and the build
	# directory is absolute.
	build_dir = api.path.abs_to_path(entry["directory"])
	abspath = api.path.realpath(api.path.join(build_dir, entry["file"]))
	try:
	path = api.path.abs_to_path(abspath)
	except ValueError:
	# This happens if `abspath` is not rooted in one of the path
	# module's known paths, which is the case for paths in `/bin`,
	# for example. That also implies that the path isn't in the
	# build directory, so skip it.
	return False

	if build_dir in path.parents:
	return False

	if "third_party" in entry["file"].split(api.path.sep):
	return False
	return True

	compdb_filtered = [entry for entry in compdb if keep_in_compdb(entry)]

	with api.context(cwd=build_results.checkout.root_dir):
	api.step(
	"run clang-doc",
	[
	build_results.tool("clang-doc"),
	"--output",
	docs_dir.joinpath("public", "cpp"),
	"--public",
	"--format=html",
	api.json.input(compdb_filtered),
	],
	)


	def GenTests(api):
	def properties():
	return api.properties(
	remote="https://fuchsia.googlesource.com/integration",
	fint_params_path="specs/firebase-docs.textproto",
	manifest="flower",
	output_name="all",
	)

	yield (api.buildbucket_util.test("firebase_docs") + properties())
	yield (
	api.buildbucket_util.test("firebase_docs_failing_rustdoc")
	+ properties()
	+ api.step_data("rustdoc.build docs.rustdoc //src/target:lib", retcode=1)
	)