recipes/contrib/docsgen.py

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for generating docs for uploading Fuchsia.dev."""

from PB.recipes.fuchsia.contrib.docsgen import InputProperties

PYTHON_VERSION_COMPATIBILITY = "PY3"

DEPS = [
    "fuchsia/build",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/gerrit",
    "fuchsia/git",
    "fuchsia/release",
    "recipe_engine/archive",
    "recipe_engine/buildbucket",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

REFERENCE_DOCS_REPOSITORY = "https://fuchsia.googlesource.com/reference-docs"
REFERENCE_DOCS_PROJECT = "reference-docs"

PROPERTIES = InputProperties


def RunSteps(api, props):
    commit_ref = api.buildbucket.build.input.gitiles_commit.ref
    checkout = api.checkout.fuchsia_with_options(
        manifest=props.manifest,
        remote=props.remote,
        is_release_version=(
            bool(commit_ref) and commit_ref.startswith("refs/heads/releases/")
        ),
    )

    build_results = api.build.with_options(
        checkout=checkout, fint_params_path=props.fint_params_path
    )

    with api.step.nest("checkout reference-docs"):
        reference_docs_dir, ref_doc_sha = api.git.checkout(REFERENCE_DOCS_REPOSITORY)
        release_branch = checkout.release_branch

        if release_branch:
            with api.context(cwd=reference_docs_dir):
                branch_exists = api.git.get_remote_branch_head(
                    step_name="check remote_branch exists",
                    url=REFERENCE_DOCS_REPOSITORY,
                    branch=release_branch,
                )
                if not branch_exists and not props.dry_run:
                    api.gerrit.create_branch(
                        name="create %s" % release_branch,
                        ref=release_branch,
                        revision=ref_doc_sha,
                        project=REFERENCE_DOCS_PROJECT,
                    )
                api.git.fetch(REFERENCE_DOCS_REPOSITORY, refspec=release_branch)
                api.git.raw_checkout(ref="FETCH_HEAD")

    gen_docs_dict = api.file.read_json(
        "read generated_docs.json", build_results.build_dir.join("generated_docs.json")
    )
    for doc_params in gen_docs_dict:
        with api.step.nest(doc_params["name"]):
            gen_doc(
                api,
                build_results.build_dir,
                doc_params,
                reference_docs_dir,
            )

    with api.context(cwd=reference_docs_dir):
        if props.dry_run:
            api.step.empty("dry run - skipping git push")
        else:
            api.git.push(refs=["HEAD:%s" % (checkout.release_branch or "main")])


def gen_doc(api, build_dir, doc_params, reference_docs_dir):
    """Copy output reference documentation to reference docs repo.

    This is the generic function to copy output reference documentation
    to reference docs repo. Reference documentation is generated from tools/docsgen
    and may or may not be archived. This function decompresses the output if needed
    and attempts to check in any differences via git.

    Args:
        build_dir (Path): Path to build directory from build_with_options step.
        doc_params (dict): Dictionary of relevant paths for reference doc.
        reference_docs_dir (Path): Path to reference docs root directory.
    """
    doc_name = doc_params["name"]
    out_dir = api.path.mkdtemp(doc_name)

    if "archive" in doc_params:
        out_dir = out_dir.join("archive")
        arch_path = build_dir.join(doc_params["archive"]["origin_file"])
        api.path.mock_add_paths(arch_path)
        if not api.path.exists(arch_path):
            return  # pragma: no cover

        # Decompress file to get all ref docs and handle base folder if exists
        api.archive.extract("decompress %s" % doc_name, arch_path, out_dir)
        if "base_folder" in doc_params["archive"]:
            output_path = out_dir.join(doc_params["archive"]["base_folder"])
        else:
            output_path = out_dir
    else:
        for origin_file in doc_params["origin_files"]:
            basename = api.path.basename(origin_file)
            api.file.copy(
                name="copy %s" % origin_file,
                source=build_dir.join(origin_file),
                dest=out_dir.join(basename),
            )
        output_path = out_dir

    reference_docs_outdir = reference_docs_dir.join(doc_params["dest_folder"])
    # Clear the prior ref docs and move results in.
    api.file.rmtree(
        name="clear reference-docs/%s" % doc_params["dest_folder"],
        source=reference_docs_outdir,
    )
    api.file.move(
        name="move %s to reference-docs/%s" % (doc_name, doc_params["dest_folder"]),
        source=output_path,
        dest=reference_docs_outdir,
    )
    with api.context(cwd=reference_docs_dir):
        # Add all modified files to git cache.
        api.git.add(add_all=True)
        # Only commit and push if there's a diff, otherwise commit fails.
        commit_message = "[%s] Update reference docs" % doc_name
        if api.git.diff(
            ref_base=None, cached=True, exit_code=True, ok_ret="any"
        ).retcode:
            api.git.commit(commit_message)


def GenTests(api):
    def properties(**kwargs):
        props = {
            "remote": "https://fuchsia.googlesource.com/integration",
            "fint_params_path": "docsgen.textproto",
            "manifest": "flower",
        }
        props.update(kwargs)
        return api.properties(**props)

    json_contents = [
        {
            "name": "driversdoc",
            "origin_files": ["all_drivers_doc.json"],
            "dest_folder": "sdk/drivers",
        },
        {
            "name": "clidoc",
            "archive": {
                "origin_file": "host_x64/obj/tools/docsgen/clidoc_out.tar.gz",
                "base_folder": "clidoc",
            },
            "dest_folder": "sdk/tools/clidoc",
        },
        {
            "name": "clidoc2",
            "archive": {
                "origin_file": "host_x64/obj/tools/docsgen/clidoc_out.tar.gz",
            },
            "dest_folder": "sdk/tools/clidoc2",
        },
    ]

    yield (
        api.buildbucket_util.test("basic")
        + properties()
        + api.step_data("read generated_docs.json", api.file.read_json(json_contents))
        + api.step_data("driversdoc.git diff", retcode=1)
    )

    yield (
        api.buildbucket_util.test(
            "existing_release_branch", repo="fuchsia", git_ref="refs/heads/releases/f5"
        )
        + properties()
        + api.release.ref_to_release_version(
            "releases/5.20210102.3.1", nesting="checkout.resolve release version"
        )
        + api.step_data("read generated_docs.json", api.file.read_json(json_contents))
        + api.step_data("driversdoc.git diff", retcode=1)
    )

    yield (
        api.buildbucket_util.test(
            "new_release_branch", repo="fuchsia", git_ref="refs/heads/releases/f5"
        )
        + properties()
        + api.release.ref_to_release_version(
            "releases/5.20210102.3.1", nesting="checkout.resolve release version"
        )
        + api.git.get_all_remote_branch_heads(
            "checkout reference-docs.check remote_branch exists", {}
        )
        + api.step_data("read generated_docs.json", api.file.read_json(json_contents))
        + api.step_data("driversdoc.git diff", retcode=1)
    )

    yield (
        api.buildbucket_util.test("dryrun")
        + properties(dry_run=True)
        + api.step_data("read generated_docs.json", api.file.read_json(json_contents))
        + api.step_data("driversdoc.git diff", retcode=1)
    )