| # Copyright 2020 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine import recipe_api |
| |
| |
| class KytheApi(recipe_api.RecipeApi): |
| """KytheApi provides support for extracting and uploading KZIPs using Kythe tools.""" |
| |
| def __init__(self, *args, **kwargs): |
| super().__init__(*args, **kwargs) |
| self.kythe_dir = None |
| self.kythe_libs_dir = None |
| |
| def _ensure(self): |
| if self.kythe_dir and self.kythe_libs_dir: |
| return |
| |
| # Fetch kythe binaries from CIPD |
| start_dir = self.m.path.start_dir |
| kythe_dir = start_dir.join("kythe") |
| kythe_libs_dir = start_dir.join("kythe-libs") |
| # Can't use parens around with statement arguments. |
| with self.m.step.nest("ensure kythe"): |
| with self.m.context(infra_steps=True): |
| pkgs = self.m.cipd.EnsureFile() |
| pkgs.add_package("fuchsia/third_party/kythe", "version:1.0.3", "kythe") |
| pkgs.add_package( |
| "fuchsia/third_party/kythe-libs/linux-amd64", "latest", "kythe-libs" |
| ) |
| self.m.cipd.ensure(start_dir, pkgs) |
| self.kythe_dir = kythe_dir |
| self.kythe_libs_dir = kythe_libs_dir |
| |
| def _merge_kzips(self, name, kzips, output_path): |
| cmd = [ |
| self.kythe_dir.join("tools", "kzip"), |
| "merge", |
| "-encoding", |
| "PROTO", |
| "-output", |
| output_path, |
| ] |
| for kzip in kzips: |
| cmd.append(str(kzip)) |
| self.m.step(name, cmd) |
| |
| def _extract_cxx_kzips(self, compdb_path): |
| self.m.step( |
| "extract C++ kzips", |
| [ |
| self.kythe_dir.join("tools", "runextractor"), |
| "compdb", |
| "-extractor", |
| self.kythe_dir.join("extractors", "cxx_extractor"), |
| "-path", |
| compdb_path, |
| ], |
| # TODO(fxbug.dev/56412): Address the ~3% of ~30k compile |
| # commands in fuchsia.git that fail to extract with kythe's |
| # tool. Allow non-zero exit status in case some compilations |
| # fail, because `runextractor` produces valid output for all |
| # the commands that succeed and returns exit code 1 if any fail. |
| ok_ret=(0, 1), |
| ) |
| |
| def _extract_rust_kzips(self, build_dir, kzip_dir): |
| self.m.step( |
| "extract Rust kzips", |
| [ |
| self.kythe_dir.join("extractors", "fuchsia_rust_extractor"), |
| "--basedir", |
| build_dir, |
| "--inputdir", |
| build_dir.join("save-analysis-temp"), |
| "--output", |
| kzip_dir, |
| "--revisions", |
| "unknown", |
| ], |
| ) |
| |
| def _validate(self, kzip_path): |
| info = self.m.step( |
| "validate kzip", |
| [self.kythe_dir.join("tools", "kzip"), "info", "--input", kzip_path], |
| stdout=self.m.json.output(), |
| ).stdout |
| if info is None: |
| raise self.m.step.StepFailure( |
| "kzip validation failed: no stdout from `kzip info`" |
| ) |
| errors = info.get("critical_kzip_errors", []) |
| if errors: |
| raise self.m.step.StepFailure( |
| f"kzip validation failed: {self.m.json.dumps(errors)}" |
| ) |
| |
| def extract_and_upload( |
| self, |
| checkout_dir, |
| build_dir, |
| corpus, |
| gcs_bucket, |
| gcs_filename, |
| langs=("cxx", "rust"), |
| compdb_path=None, |
| vnames_path=None, |
| ): |
| if compdb_path is None: |
| compdb_path = build_dir.join("compile_commands.json") |
| |
| self._ensure() |
| |
| # Run `runextractor` on compile_commands.json |
| kzip_dir = self.m.path.start_dir.join("kythe-output") |
| self.m.file.ensure_directory("kzip directory", kzip_dir) |
| env = { |
| "KYTHE_CORPUS": corpus, |
| "KYTHE_OUTPUT_DIRECTORY": kzip_dir, |
| "KYTHE_ROOT_DIRECTORY": checkout_dir, |
| "LD_LIBRARY_PATH": self.kythe_libs_dir, |
| } |
| if vnames_path: |
| env["KYTHE_VNAMES"] = vnames_path |
| with self.m.context(cwd=build_dir, env=env): |
| if "cxx" in langs: |
| self._extract_cxx_kzips(compdb_path) |
| if "rust" in langs: |
| self._extract_rust_kzips(build_dir, kzip_dir) |
| |
| # Merge kzips |
| kzips = self.m.file.glob_paths("glob kzips", kzip_dir, "*.kzip") |
| kzip_path = kzip_dir.join("merged.kzip") |
| |
| # First merge kzips in groups of 10k, to avoid overflowing the allowed |
| # buffer for command line arguments. |
| partitions = 0 |
| partition_size = 10000 |
| intermediary_kzips = [] |
| while partitions * partition_size < len(kzips): |
| already_merged = partitions * partition_size |
| to_be_merged = already_merged + partition_size |
| intermediary_kzip = kzip_dir.join(f"intermediate_{partitions}.kzip") |
| intermediary_kzips.append(intermediary_kzip) |
| self._merge_kzips( |
| f"merge next {partition_size} kzips (total {to_be_merged})", |
| kzips[already_merged:to_be_merged], |
| intermediary_kzip, |
| ) |
| partitions += 1 |
| |
| # Then merge those resulting kzips together. |
| self._merge_kzips("merge into final kzip", intermediary_kzips, kzip_path) |
| |
| self._validate(kzip_path) |
| |
| # Upload merged kzip to the GCS bucket where kythe expects to find it, |
| # so that it can be indexed |
| self.m.gsutil.upload( |
| gcs_bucket, # bucket |
| kzip_path, # source |
| gcs_filename, # dest |
| name="upload kzip", |
| ) |