recipe_modules/kythe/api.py

# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from recipe_engine import recipe_api


class KytheApi(recipe_api.RecipeApi):
    """KytheApi provides support for extracting and uploading KZIPs using Kythe tools."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.kythe_dir = None
        self.kythe_libs_dir = None

    def _ensure(self):
        if self.kythe_dir and self.kythe_libs_dir:
            return

        # Fetch kythe binaries from CIPD
        start_dir = self.m.path["start_dir"]
        kythe_dir = start_dir.join("kythe")
        kythe_libs_dir = start_dir.join("kythe-libs")
        # Can't use parens around with statement arguments.
        with self.m.step.nest("ensure kythe"):
            with self.m.context(infra_steps=True):
                pkgs = self.m.cipd.EnsureFile()
                pkgs.add_package("fuchsia/third_party/kythe", "version:1.0.3", "kythe")
                pkgs.add_package(
                    "fuchsia/third_party/kythe-libs/linux-amd64", "latest", "kythe-libs"
                )
                self.m.cipd.ensure(start_dir, pkgs)
                self.kythe_dir = kythe_dir
                self.kythe_libs_dir = kythe_libs_dir

    def _merge_kzips(self, name, kzips, output_path):
        cmd = [
            self.kythe_dir.join("tools", "kzip"),
            "merge",
            "-encoding",
            "PROTO",
            "-output",
            output_path,
        ]
        for kzip in kzips:
            cmd.append(str(kzip))
        self.m.step(name, cmd)

    def _extract_cxx_kzips(self, compdb_path):
        self.m.step(
            "extract C++ kzips",
            [
                self.kythe_dir.join("tools", "runextractor"),
                "compdb",
                "-extractor",
                self.kythe_dir.join("extractors", "cxx_extractor"),
                "-path",
                compdb_path,
            ],
            # TODO(fxbug.dev/56412): Address the ~3% of ~30k compile
            # commands in fuchsia.git that fail to extract with kythe's
            # tool. Allow non-zero exit status in case some compilations
            # fail, because `runextractor` produces valid output for all
            # the commands that succeed and returns exit code 1 if any fail.
            ok_ret=(0, 1),
        )

    def _extract_rust_kzips(self, build_dir, kzip_dir):
        self.m.step(
            "extract Rust kzips",
            [
                self.kythe_dir.join("extractors", "fuchsia_rust_extractor"),
                "--basedir",
                build_dir,
                "--inputdir",
                build_dir.join("save-analysis-temp"),
                "--output",
                kzip_dir,
                "--revisions",
                "unknown",
            ],
        )

    def _validate(self, kzip_path):
        info = self.m.step(
            "validate kzip",
            [self.kythe_dir.join("tools", "kzip"), "info", "--input", kzip_path],
            stdout=self.m.json.output(),
        ).stdout
        if info is None:
            raise self.m.step.StepFailure(
                "kzip validation failed: no stdout from `kzip info`"
            )
        errors = info.get("critical_kzip_errors", [])
        if errors:
            raise self.m.step.StepFailure(
                "kzip validation failed: %s" % self.m.json.dumps(errors)
            )

    def extract_and_upload(
        self,
        checkout_dir,
        build_dir,
        corpus,
        gcs_bucket,
        gcs_filename,
        langs=("cxx", "rust"),
        compdb_path=None,
    ):
        if compdb_path is None:
            compdb_path = build_dir.join("compile_commands.json")

        self._ensure()

        # Run `runextractor` on compile_commands.json
        kzip_dir = self.m.path["start_dir"].join("kythe-output")
        self.m.file.ensure_directory("kzip directory", kzip_dir)
        with self.m.context(
            cwd=build_dir,
            env={
                "KYTHE_ROOT_DIRECTORY": checkout_dir,
                "KYTHE_OUTPUT_DIRECTORY": kzip_dir,
                "KYTHE_CORPUS": corpus,
                "LD_LIBRARY_PATH": self.kythe_libs_dir,
            },
        ):
            if "cxx" in langs:
                self._extract_cxx_kzips(compdb_path)
            if "rust" in langs:
                self._extract_rust_kzips(build_dir, kzip_dir)

        # Merge kzips
        kzips = self.m.file.glob_paths("glob kzips", kzip_dir, "*.kzip")
        kzip_path = kzip_dir.join("merged.kzip")

        # First merge kzips in groups of 10k, to avoid overflowing the allowed
        # buffer for command line arguments.
        partitions = 0
        partition_size = 10000
        intermediary_kzips = []
        while partitions * partition_size < len(kzips):
            already_merged = partitions * partition_size
            to_be_merged = already_merged + partition_size
            intermediary_kzip = kzip_dir.join("intermediate_%s.kzip" % partitions)
            intermediary_kzips.append(intermediary_kzip)
            self._merge_kzips(
                "merge next %s kzips (total %s)" % (partition_size, to_be_merged),
                kzips[already_merged:to_be_merged],
                intermediary_kzip,
            )
            partitions += 1

        # Then merge those resulting kzips together.
        self._merge_kzips("merge into final kzip", intermediary_kzips, kzip_path)

        self._validate(kzip_path)

        # Upload merged kzip to the GCS bucket where kythe expects to find it,
        # so that it can be indexed
        self.m.gsutil.upload(
            gcs_bucket,  # bucket
            kzip_path,  # source
            gcs_filename,  # dest
            name="upload kzip",
        )