# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for generating docs for upload to Firebase."""

from recipe_engine.config import Enum, List
from recipe_engine.recipe_api import Property

import hashlib

DEPS = [
    "fuchsia/build",
    "fuchsia/checkout",
    "fuchsia/git",
    "fuchsia/status_check",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/python",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

TARGETS = ["arm64", "x64"]
BUILD_TYPES = ["debug", "release", "thinlto", "lto"]
GENERATORS = ["dartdoc", "rustdoc", "fidldoc", "clangdoc"]

REFERENCE_DOCS_REPOSITORY = "https://fuchsia.googlesource.com/reference-docs"
API_DOCS_RESOURCES_REPOSITORY = "https://fuchsia.googlesource.com/api-docs-resources"

PROPERTIES = {
    "board": Property(kind=str, help="Board to build"),
    "build_type": Property(kind=Enum(*BUILD_TYPES), help="The build type"),
    "dry_run": Property(
        kind=bool, help="Whether to upload docs to firebase.", default=False
    ),
    "exclude_images": Property(
        kind=bool, help="Whether to skip building Fuchsia images"
    ),
    # TODO(olivernewman): Convert to protobuf properties and make a Generator
    # enum.
    "generators": Property(
        kind=List(basestring), help="Packages to build", default=GENERATORS
    ),
    "manifest": Property(kind=str, help="Jiri manifest to use", default="topaz/topaz"),
    "output_name": Property(
        kind=str,
        help="Name of top-level output reference-docs directory",
        default="all",
    ),
    "product": Property(kind=str, help="Product to build"),
    "remote": Property(
        kind=str,
        help="Remote manifest repository",
        default="https://fuchsia.googlesource.com/integration",
    ),
    "fint_params_path": Property(kind=str, help="Path to a fint params file"),
    "target": Property(kind=Enum(*TARGETS), help="Target to build"),
}

DARTDOC_PUBSPEC = """name: Fuchsia
homepage: https://fuchsia-docs.firebaseapp.com/dart
description: API documentation for fuchsia
dependencies:
"""

DARTDOC_COMMIT_MESSAGE = "[dartdoc] Updating dart reference docs"
FIDLDOC_COMMIT_MESSAGE = "[fidldoc] Updating fidl reference docs"


def gen_dartdoc(api, out_dir, docs_dir, cipd_dir, output_name, dry_run):
    """Generate dartdoc output.

    Dartdoc runs on a single package, but has the capability to generate docs for all
    dependencies. Thus, to generate Dart documentation for Fuchsia, we first generate
    a 'fake' package that lists the libraries we want documented. We then run `pub`
    over that new package to fetch the dependencies, and finally `dartdoc` to generate
    documentation for it all.

    Args:
      out_dir (Path) - The output directory for generated files.
      docs_dir (Path) - The output directory for documentation.
      cipd_dir (Path) - The cipd directory.
      output_name (str) - Output name of the directory.
      dry_run (str) - Run but don't push.
    """
    dart_packages_path = api.path["start_dir"].join("sdk", "dart")
    api.path.mock_add_paths(dart_packages_path)
    # If either dartdoc or dart packages path doesn't exist, we didn't checkout the
    # repository of interest and so we won't generate dart docs on this run.
    if not api.path.exists(dart_packages_path):
        return  # pragma: no cover

    # Make a temporary docs dir to be pushed to firebase.
    api.file.ensure_directory("create lib dir", out_dir.join("lib"))

    # Build .packages and lib.dart importing all packages.
    dart_imports_content = "library Fuchsia;\n"
    dart_pubspec_content = DARTDOC_PUBSPEC

    # Gather documentable dart packages.
    dart_packages = [
        api.path.basename(p)
        for p in api.file.listdir(
            "list dart packages",
            dart_packages_path,
            test_data=("fuchsia", "sdk", "other"),
        )
    ]

    api.path.mock_add_paths(dart_packages_path.join("fuchsia", "lib"))
    api.path.mock_add_paths(dart_packages_path.join("fuchsia", "pubspec.yaml"))
    api.path.mock_add_paths(dart_packages_path.join("sdk", "lib"))

    for package in dart_packages:
        if not api.path.exists(dart_packages_path.join(package, "lib")):
            continue

        pubspec_path = dart_packages_path.join(package, "pubspec.yaml")
        if not api.path.exists(pubspec_path):
            continue

        pubspec = api.python(
            "load %s pubspec.yaml" % package,
            api.resource("parse_yaml.py"),
            args=[pubspec_path],
            stdout=api.json.output(),
        ).stdout

        if not pubspec or pubspec["name"] != package:
            continue  # pragma: no cover

        dart_pubspec_content += " %s:\n    path: %s/\n" % (
            package,
            dart_packages_path.join(package),
        )
        package_imports = [
            api.path.basename(i)
            for i in api.file.listdir(
                "list %s packages" % package, dart_packages_path.join(package, "lib")
            )
            if api.path.basename(i).endswith(".dart")
        ]
        for i in package_imports:
            dart_imports_content += "import 'package:%s/%s';\n" % (package, i)

    # Build package pubspec.yaml depending on all desired source packages.
    api.file.write_text(
        "write pubspec.yaml", out_dir.join("pubspec.yaml"), dart_pubspec_content
    )
    api.file.write_text(
        "write lib.dart", out_dir.join("lib", "lib.dart"), dart_imports_content
    )

    # Run pub over this package to fetch deps.
    with api.context(cwd=out_dir):
        api.step("pub", [cipd_dir.join("dart-sdk", "bin", "pub"), "get"])

    # Record the version in case changes to dartdoc cause a regression.
    with api.context(cwd=out_dir):
        api.step(
            "dartdoc version",
            [cipd_dir.join("dart-sdk", "bin", "dartdoc"), "--version"],
        )

    excluded_packages = [
        "Dart",  # Dart SDK
        "logging",  # used by Fuchsia logger
    ]
    # Run dartdoc over this package and generate HTML for firebase hosting.
    # TODO(jdkoren): remove once Fuchsia Devsite has suitable Dart reference docs.
    with api.context(cwd=out_dir):
        api.step(
            "dartdoc",
            [
                cipd_dir.join("dart-sdk", "bin", "dartdoc"),
                "--auto-include-dependencies",
                "--exclude-packages",
                ",".join(excluded_packages),
                "--output",
                docs_dir.join("public", "dart"),
            ],
        )

    # Run dartdoc over this package and generate Markdown for Fuchsia Devsite.
    # Also generate a TOC from the generated index.
    md_docs_dir = api.path["start_dir"].join("dartdoc_out_md")
    with api.context(cwd=out_dir):
        api.step(
            "dartdoc devsite",
            [
                cipd_dir.join("dart-sdk", "bin", "dartdoc"),
                "--auto-include-dependencies",
                "--exclude-packages",
                ",".join(excluded_packages),
                "--output",
                md_docs_dir,
                "--format",
                "md",
            ],
        )
        api.python(
            "generate _toc.yaml file",
            script=api.resource("generate_dart_toc.py"),
            args=[
                "--index-file",
                md_docs_dir.join("index.json"),
                "--outfile",
                md_docs_dir.join("_toc.yaml"),
            ],
        )

    # Markdown docs are committed to reference docs repo. Checkout the repo here
    # to minimize the likelihood of a race with another bot trying to commit to
    # the same repo between this bot's checkout and commit.
    reference_docs_dir = api.path["start_dir"].join("reference-docs")
    with api.step.nest("checkout reference-docs"):
        api.git.checkout(REFERENCE_DOCS_REPOSITORY, path=reference_docs_dir)
    with api.context(cwd=reference_docs_dir):
        reference_docs_dart_dir = reference_docs_dir.join(output_name, "dart")
        # Clear the prior dart docs and move results in.
        api.file.rmtree(
            name="clear reference-docs/out/dart", source=reference_docs_dart_dir,
        )
        api.file.move(
            name="move docs to reference-docs/out/dart",
            source=md_docs_dir,
            dest=reference_docs_dart_dir,
        )
        # Remove artifacts of dartdoc that will not be used.
        api.file.remove(
            name="remove reference-docs/out/dart/index.json",
            source=reference_docs_dart_dir.join("index.json"),
        )
        api.file.remove(
            name="remove reference-docs/out/dart/categories.json",
            source=reference_docs_dart_dir.join("categories.json"),
        )
        api.file.remove(
            name="remove reference-docs/out/dart/__404error.md",
            source=reference_docs_dart_dir.join("__404error.md"),
        )

        # Add all modified files to git cache.
        api.git.add(add_all=True)
        # Only commit and push if there's a diff, otherwise commit fails.
        if api.git.diff(
            ref_base=None, cached=True, exit_code=True, ok_ret="any"
        ).retcode:
            api.git.commit(DARTDOC_COMMIT_MESSAGE)
            if not dry_run:
                api.git.push(refs=["HEAD:master"])


def gen_rustdoc(api, docs_dir, build_dir):
    """Generate rust output.

    The rustdoc script runs on GN targets. We find the Rust GN targets by finding
    all targets that generate a Cargo.toml file, and use those.

    Args:
        docs_dir (Path): The output directory for documentation.
        build_dir (Path): The build directory.
    """

    project = api.file.read_json("read project.json", build_dir.join("project.json"))

    for target in ["lib", "bin"]:
        hsh = hashlib.sha1("topaz/target:%s" % target).hexdigest()
        cargo_path = build_dir.join("cargo", hsh, "Cargo.toml")
        api.path.mock_add_paths(cargo_path)

    skipped = []
    for target, info in project["targets"].iteritems():
        # Look for Rust targets only.
        if "crate_root" not in info:
            continue

        # TODO(tmandry): Find a way that doesn't involve duplicating this logic.
        hsh = hashlib.sha1(target[2:]).hexdigest()
        cargo_toml_path = build_dir.join("cargo", hsh, "Cargo.toml")
        if not api.path.exists(cargo_toml_path):
            skipped.append(target)
            continue

        try:
            with api.context(
                env={
                    "FUCHSIA_DIR": api.path["start_dir"],
                    "FUCHSIA_BUILD_DIR": build_dir,
                }
            ):
                api.step(
                    "rustdoc %s" % target,
                    [
                        api.path["start_dir"].join(
                            "tools", "devshell", "contrib", "lib", "rust", "rustdoc.py"
                        ),
                        cargo_toml_path,
                        "--no-deps",
                        "--out-dir",
                        build_dir,
                    ],
                )
        except api.step.StepFailure:
            pass

    # Move the output to the docs directory.
    step_result = api.step(
        "move output to docs",
        [
            "mv",
            api.path["start_dir"].join("out", "cargo_target", "x86_64-fuchsia", "doc"),
            docs_dir.join("public", "rust"),
        ],
    )

    step_result.presentation.logs["skipped"] = skipped


def gen_fidldoc(api, build_dir, output_name, dry_run):
    """Generate fidl output.

    The fidldoc tool runs on the all_fidl_json.txt file. Pushes the resulting
    docs to the given docs repository.

    Args:
      docs_dir (Path) - The output directory for documentation (repository).
      build_dir (Path) - The build directory.
      output_name (str) - Output name of the directory.
      dry_run (str) - Run but don't push.
    """
    out_dir = api.path["start_dir"].join("fidldoc_out")
    all_fidl_json_txt = build_dir.join("all_fidl_json.txt")
    all_fidl_json = api.file.read_text(
        "read all_fidl_json.txt", all_fidl_json_txt
    ).splitlines()

    with api.context(cwd=build_dir):
        # Cannot use tool_paths.json here, since it maps to host_x64, which
        # doesn't work for fidldoc because of the accompanying fidldoc.config.json
        # that is only in host-tools.
        fidldoc_path = build_dir.join("host-tools/fidldoc")
        api.step(
            "run fidldoc",
            [fidldoc_path, "--verbose", "--path", "/reference/fidl/", "--out", out_dir]
            + all_fidl_json,
        )

    # Push resulting docs to the reference-docs repository.

    # The fidldocs get checked into their own repository. Checking it out here reduces the likelihood
    # of a race with another bot trying to commit in between this bot's checkout and commit steps.
    reference_docs_dir = api.path["start_dir"].join("reference-docs")
    with api.step.nest("checkout reference-docs"):
        api.git.checkout(REFERENCE_DOCS_REPOSITORY, path=reference_docs_dir)

    with api.context(cwd=reference_docs_dir):
        # Clear the repository and move results in.
        api.file.rmtree(
            name="clear reference-docs/out/fidl",
            source=reference_docs_dir.join(output_name, "fidl"),
        )
        api.file.move(
            name="Move docs to reference-docs/out/fidl",
            source=out_dir,
            dest=reference_docs_dir.join(output_name, "fidl"),
        )
        # Add all modified files to git cache.
        api.git.add(add_all=True)
        # Only commit and push if there's a diff, otherwise commit fails.
        if api.git.diff(
            ref_base=None, cached=True, exit_code=True, ok_ret="any"
        ).retcode:
            api.git.commit(FIDLDOC_COMMIT_MESSAGE)
            if not dry_run:
                api.git.push(refs=["HEAD:master"])


def gen_clang_doc(api, docs_dir, build_dir, gn_results):
    """Generate clang-doc output.

    clang-doc runs on the translation units specified in the compilation database
    file. This file will be generated by gn_results after filtering unwanted
    directories or files. clang-doc will output documentation files directly in
    docs_dir.

    Args:
      docs_dir (Path) - The output directory for documentation.
      build_dir (Path) - The build directory.
      gn_results (Object) - Result of a `gn gen` invocation in the fuchsia build.
    """
    with api.step.nest("filter compile commands"):
        white_list_dirs = ["third_party"]
        compile_commands = gn_results.filtered_compdb(white_list_dirs)

    with api.context(cwd=build_dir):
        api.step(
            "run clang-doc",
            [
                gn_results.tool("clang-doc"),
                "--output",
                docs_dir.join("public", "cpp"),
                "--public",
                "--format=html",
                compile_commands,
            ],
        )


def RunSteps(
    api,
    board,
    build_type,
    dry_run,
    exclude_images,
    generators,
    manifest,
    output_name,
    product,
    remote,
    target,
    fint_params_path,
):
    cipd_dir = api.path["start_dir"].join("cipd")
    node_modules_dir = cipd_dir.join("node_modules")
    with api.step.nest("ensure_packages"):
        with api.context(infra_steps=True):
            pkgs = api.cipd.EnsureFile()
            pkgs.add_package("infra/nodejs/nodejs/${platform}", "latest")
            if "dartdoc" in generators:
                pkgs.add_package("dart/dart-sdk/${platform}", "dev")
            api.cipd.ensure(cipd_dir, pkgs)

            # firebase-tools expects to live in the node_modules subdir of where nodejs is installed.
            pkgs = api.cipd.EnsureFile()
            pkgs.add_package("infra/npm/firebase-tools", "latest")
            api.cipd.ensure(node_modules_dir, pkgs)

    resources_dir = api.path["start_dir"].join("api-docs-resources")
    with api.step.nest("checkout api docs"):
        api.git.checkout(API_DOCS_RESOURCES_REPOSITORY, path=resources_dir)

    checkout_root = api.path["start_dir"]
    with api.step.nest("checkout fuchsia"):
        checkout = api.checkout.fuchsia_with_options(
            path=checkout_root, manifest=manifest, remote=remote,
        )

    build_results = api.build.with_options(
        checkout=checkout,
        target=target,
        build_images=not exclude_images,
        build_type=build_type,
        board=board,
        product=product,
        fint_params_path=fint_params_path,
    )

    out_dir = api.path["start_dir"].join("docs_out")
    docs_dir = api.path["start_dir"].join("firebase")

    api.file.rmtree("remove old docs", docs_dir)
    api.file.copytree("copy resources", resources_dir, docs_dir)

    if "clangdoc" in generators:
        with api.step.nest("clangdoc"):
            gen_clang_doc(
                api, docs_dir, api.path["start_dir"], build_results.gn_results
            )
    if "dartdoc" in generators:
        with api.step.nest("dartdoc"):
            gen_dartdoc(api, out_dir, docs_dir, cipd_dir, output_name, dry_run)
    if "fidldoc" in generators:
        with api.step.nest("fidldoc"):
            gen_fidldoc(api, build_results.build_dir, output_name, dry_run)
    if "rustdoc" in generators:
        with api.step.nest("rustdoc"):
            gen_rustdoc(api, docs_dir, build_results.build_dir)

    # Only deploy if running the default 'all' one, to avoid deploying partial docs to firebase.
    if not dry_run and output_name == "all":
        with api.context(cwd=docs_dir, env={"PATH": cipd_dir.join("bin")}):
            api.step(
                "firebase deploy",
                [
                    node_modules_dir.join(".bin", "firebase"),
                    "deploy",
                    "--only",
                    "hosting",
                    "--debug",
                ],
            )


def GenTests(api):
    base = (
        api.buildbucket.ci_build(git_repo="https://fuchsia.googlesource.com/topaz",)
        + api.properties(
            board="boards/x64.gni",
            build_type="release",
            exclude_images=True,
            product="products/core.gni",
            fint_params_path="specs/firebase-docs.textproto",
            target="x64",
        )
        + api.step_data(
            "dartdoc.load fuchsia pubspec.yaml",
            stdout=api.json.output({"name": "fuchsia"}),
        )
        + api.step_data(
            "dartdoc.list fuchsia packages", api.file.listdir(["fuchsia.dart"])
        )
        + api.step_data("dartdoc.git diff", retcode=1)
        + api.step_data(
            "rustdoc.read project.json",
            api.file.read_json(
                json_content={
                    "targets": {
                        "//topaz/target:lib": {
                            "type": "rust_library",
                            "crate_root": "src/lib.rs",
                        },
                        "//topaz/target:bin": {
                            "type": "executable",
                            "crate_root": "src/main.rs",
                        },
                        "//topaz/not_target:not_rust": {"type": "staticlib"},
                        "//topaz/target:missing_cargo": {
                            "type": "rust_library",
                            "crate_root": "src/lib.rs",
                        },
                        "//topaz/not_target:no_type": {},
                    }
                }
            ),
        )
        + api.step_data(
            "fidldoc.read all_fidl_json.txt", api.raw_io.output("foo.fidl\nbar.fidl\n")
        )
        + api.step_data("fidldoc.git diff", retcode=1)
    )
    yield (api.status_check.test("firebase_docs") + base)
    yield (
        api.status_check.test("firebase_docs_failing_rustdoc")
        + base
        + api.step_data("rustdoc.rustdoc //topaz/target:lib", retcode=1)
    )