recipe_modules/autocorrelator/api.py - infra/recipes - Git at Google

 # Copyright 2021 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 from recipe_engine import recipe_api

 from RECIPE_MODULES.fuchsia.utils import pluralize

 from PB.go.chromium.org.luci.buildbucket.proto import builder as builder_pb2

 AUTOCORRELATOR_PATH = "fuchsia/infra/autocorrelator/${platform}"
 AUTOCORRELATOR_VERSION = "git_revision:3b19be8ad77b1927378b00eaa4aa12bc24c27184"
 AUTOCORRELATOR_MAX_SUMMARY_SIZE = 750
 AUTOCORRELATOR_HEADER = "This failure may be a false rejection."
 AUTOCORRELATOR_FOOTER = "Original failure continues below."


 class AutocorrelatorApi(recipe_api.RecipeApi):
     """APIs for finding correlations between failures in Fuchsia CI/CQ."""

     def __init__(self, props, *args, **kwargs):
         super(AutocorrelatorApi, self).__init__(*args, **kwargs)
         self.ci_bucket = props.ci_bucket
         self.ci_builder = props.ci_builder
         self.findings = []

     def check_ci(
         self,
         step_name,
         base_commit,
         exception,
         summary_markdown,
         score_threshold=0.95,
     ):
         """Compare summary markdown similarity to a CI-like builder. Record a
         finding if it meets the score threshold.

         Args:
             step_name (str): Name of the step.
             base_commit (str): Base commit as sha1 to check against `builder`.
             exception (Exception): The exception corresponding to the summary
                 markdown.
             summary_markdown (str): Summary markdown to compare.
             score_threshold (float): Record a finding if it meets this
                 threshold.
         """
         assert self.m.buildbucket_util.is_tryjob, "invalid command in non-try"
         if not self.ci_bucket:
             return None
         with self.m.step.nest(step_name):
             args = [
                 "check-ci",
                 "-base-commit",
                 base_commit,
                 "-builder",
                 self.m.buildbucket_util.full_builder_name(
                     builder_pb2.BuilderID(
                         project=self.m.buildbucket.build.builder.project,
                         bucket=self.ci_bucket,
                         builder=self.ci_builder
                         or self.m.buildbucket.build.builder.builder,
                     )
                 ),
                 "-build-status",
                 self.m.buildbucket_util.build_status(exception),
                 "-summary-markdown-path",
                 self._summary_markdown_path(summary_markdown),
                 "-json-output",
                 self.m.json.output(),
             ]
             step = self("run autocorrelator", args)
             finding = step.json.output
             if not finding or finding["is_green"]:
                 return step
             score = finding["score"]
             if score >= score_threshold:
                 ci_build_link = self.m.buildbucket.build_url(
                     build_id=finding["build_id"]
                 )
                 step.presentation.links[finding["build_id"]] = ci_build_link
                 self.findings.append(
                     "Correlated failure found in CI: %0.2f similarity, %d git "
                     "commit distance\n\n- %s"
                     % (score, finding["commit_dist"], ci_build_link),
                 )
             return step

     def check_try(
         self,
         step_name,
         exception,
         summary_markdown,
         builder=None,
         change_num=None,
         ignore_failed_build=False,
         ignore_skipped_build=False,
         ignore_skipped_tests=False,
         score_threshold=0.95,
         build_frequency_threshold=0.6,
     ):
         """Compare summary markdown similarity to a try-like builder. Record an
         aggregate finding if there are one or more findings that meet the score
         threshold, and the frequency of findings meets the build frequency
         threshold.

         Args:
             step_name (str): Name of the step.
             exception (Exception): The exception corresponding to the summary
                 markdown.
             summary_markdown (str): Summary markdown to compare.
             builder (str or None): Fully-qualified Buildbucket builder name of
                 the try-like builder to check. If None, use the current build's
                 builder name.
             change_num (int or None): Gerrit change number. If None, use the
                 current build's change number.
             ignore_failed_build (bool): Whether to ignore try builds which
                 failed to build.
             ignore_skipped_build (bool): Whether to ignore try builds with
                 unaffected build graphs.
             ignore_skipped_tests (bool): Whether to ignore builds which skipped
                 testing.
             score_threshold (float): Count a finding as part of the aggregate if
                 its score meets this threshold.
             build_frequency_threshold (float): Record an aggregate finding if
                 the frequency of findings meets this threshold, i.e. a 0.5
                 threshold means that a finding will be recorded if
                 num_findings / num_inspected_builds >= 0.5.
         """
         assert self.m.buildbucket_util.is_tryjob, "invalid command in non-try"
         with self.m.step.nest(step_name):
             args = [
                 "check-try",
                 "-builder",
                 builder or self.m.buildbucket_util.full_builder_name(),
                 "-change-num",
                 change_num or self.m.buildbucket.build.input.gerrit_changes[0].change,
                 "-build-status",
                 self.m.buildbucket_util.build_status(exception),
                 "-summary-markdown-path",
                 self._summary_markdown_path(summary_markdown),
                 "-scrub-header",
                 AUTOCORRELATOR_HEADER,
                 "-scrub-footer",
                 AUTOCORRELATOR_FOOTER,
                 "-json-output",
                 self.m.json.output(),
             ]
             if ignore_failed_build:
                 args.append("-ignore-failed-build")
             if ignore_skipped_build:
                 args.append("-ignore-skipped-build")
             if ignore_skipped_tests:
                 args.append("-ignore-skipped-tests")
             step = self("run autocorrelator", args)
             findings = step.json.output
             if not findings:
                 return step
             findings_meeting_threshold = []
             for finding in findings:
                 if finding["score"] >= score_threshold:
                     findings_meeting_threshold.append(finding)
                     try_build_link = self.m.buildbucket.build_url(
                         build_id=finding["build_id"]
                     )
                     step.presentation.links[finding["build_id"]] = try_build_link
             build_frequency = len(findings_meeting_threshold) / float(len(findings))
             if build_frequency >= build_frequency_threshold:
                 self.findings.append(
                     "Similar failures found in try: %0.2f avg similarity in %s\n\n%s"
                     % (
                         sum(
                             (finding["score"] for finding in findings_meeting_threshold)
                         )
                         / len(findings_meeting_threshold),
                         pluralize("build", len(findings_meeting_threshold)),
                         "\n".join(
                             "- %s" % build_link
                             for build_link in step.presentation.links.itervalues()
                         ),
                     ),
                 )
             return step

     def set_properties(self, presentation):
         """Set output properties for current autocorrelator findings."""
         presentation.properties["num_autocorrelator_findings"] = len(self.findings)

     def compose_exception(self, original_exc):
         """Compose a modified StepFailure with the current autocorrelator
         findings.

         Args:
             original_exc (Exception): Exception to modify.

         Returns:
             StepFailure: containing the modified summary markdown
             including autocorrelator findings.
         """
         # If the exception is not a StepFailure then there's no consistent way
         # to obtain the original summary markdown. Likewise, if there are no
         # findings then there's no context to add to the exception.
         if not isinstance(original_exc, self.m.step.StepFailure) or not self.findings:
             return original_exc

         # Construct a summary markdown for current findings.
         autocorrelator_summary = self.m.buildbucket_util.summary_message(
             "\n\n".join(self.findings),
             "findings truncated, see autocorrelator steps for full details",
             truncate_length=(
                 AUTOCORRELATOR_MAX_SUMMARY_SIZE
                 # Include space for header and footer text.
                 - len(AUTOCORRELATOR_HEADER)
                 - len(AUTOCORRELATOR_FOOTER)
                 # Include space for newline characters between each component.
                 # See join() below.
                 - 2 * 3
             ),
             escape_markdown=False,
         )
         # Stitch together the full summary markdown.
         summary_markdown = "\n\n".join(
             (
                 AUTOCORRELATOR_HEADER,
                 autocorrelator_summary,
                 AUTOCORRELATOR_FOOTER,
                 original_exc.reason,
             )
         )

         if isinstance(original_exc, self.m.step.InfraFailure):
             return self.m.step.InfraFailure(summary_markdown)
         elif isinstance(original_exc, self.m.step.StepWarning):
             return self.m.step.StepWarning(summary_markdown)
         return self.m.step.StepFailure(summary_markdown)

     def __call__(self, step_name, args):
         return self.m.step(
             step_name, [self._autocorrelator_tool] + args, infra_step=True
         )

     def _summary_markdown_path(self, summary_markdown):
         """Write summary markdown to a file and return the Path."""
         path = self.m.path.mkstemp("summary_md")
         self.m.file.write_text("write summary markdown", path, str(summary_markdown))
         return path

     @property
     def _autocorrelator_tool(self):
         return self.m.cipd.ensure_tool(AUTOCORRELATOR_PATH, AUTOCORRELATOR_VERSION)
	# Copyright 2021 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	from recipe_engine import recipe_api

	from RECIPE_MODULES.fuchsia.utils import pluralize

	from PB.go.chromium.org.luci.buildbucket.proto import builder as builder_pb2

	AUTOCORRELATOR_PATH = "fuchsia/infra/autocorrelator/${platform}"
	AUTOCORRELATOR_VERSION = "git_revision:3b19be8ad77b1927378b00eaa4aa12bc24c27184"
	AUTOCORRELATOR_MAX_SUMMARY_SIZE = 750
	AUTOCORRELATOR_HEADER = "This failure may be a false rejection."
	AUTOCORRELATOR_FOOTER = "Original failure continues below."


	class AutocorrelatorApi(recipe_api.RecipeApi):
	"""APIs for finding correlations between failures in Fuchsia CI/CQ."""

	def __init__(self, props, args, *kwargs):
	super(AutocorrelatorApi, self).__init__(args, *kwargs)
	self.ci_bucket = props.ci_bucket
	self.ci_builder = props.ci_builder
	self.findings = []

	def check_ci(
	self,
	step_name,
	base_commit,
	exception,
	summary_markdown,
	score_threshold=0.95,
	):
	"""Compare summary markdown similarity to a CI-like builder. Record a
	finding if it meets the score threshold.

	Args:
	step_name (str): Name of the step.
	base_commit (str): Base commit as sha1 to check against `builder`.
	exception (Exception): The exception corresponding to the summary
	markdown.
	summary_markdown (str): Summary markdown to compare.
	score_threshold (float): Record a finding if it meets this
	threshold.
	"""
	assert self.m.buildbucket_util.is_tryjob, "invalid command in non-try"
	if not self.ci_bucket:
	return None
	with self.m.step.nest(step_name):
	args = [
	"check-ci",
	"-base-commit",
	base_commit,
	"-builder",
	self.m.buildbucket_util.full_builder_name(
	builder_pb2.BuilderID(
	project=self.m.buildbucket.build.builder.project,
	bucket=self.ci_bucket,
	builder=self.ci_builder
	or self.m.buildbucket.build.builder.builder,
	)
	),
	"-build-status",
	self.m.buildbucket_util.build_status(exception),
	"-summary-markdown-path",
	self._summary_markdown_path(summary_markdown),
	"-json-output",
	self.m.json.output(),
	]
	step = self("run autocorrelator", args)
	finding = step.json.output
	if not finding or finding["is_green"]:
	return step
	score = finding["score"]
	if score >= score_threshold:
	ci_build_link = self.m.buildbucket.build_url(
	build_id=finding["build_id"]
	)
	step.presentation.links[finding["build_id"]] = ci_build_link
	self.findings.append(
	"Correlated failure found in CI: %0.2f similarity, %d git "
	"commit distance\n\n- %s"
	% (score, finding["commit_dist"], ci_build_link),
	)
	return step

	def check_try(
	self,
	step_name,
	exception,
	summary_markdown,
	builder=None,
	change_num=None,
	ignore_failed_build=False,
	ignore_skipped_build=False,
	ignore_skipped_tests=False,
	score_threshold=0.95,
	build_frequency_threshold=0.6,
	):
	"""Compare summary markdown similarity to a try-like builder. Record an
	aggregate finding if there are one or more findings that meet the score
	threshold, and the frequency of findings meets the build frequency
	threshold.

	Args:
	step_name (str): Name of the step.
	exception (Exception): The exception corresponding to the summary
	markdown.
	summary_markdown (str): Summary markdown to compare.
	builder (str or None): Fully-qualified Buildbucket builder name of
	the try-like builder to check. If None, use the current build's
	builder name.
	change_num (int or None): Gerrit change number. If None, use the
	current build's change number.
	ignore_failed_build (bool): Whether to ignore try builds which
	failed to build.
	ignore_skipped_build (bool): Whether to ignore try builds with
	unaffected build graphs.
	ignore_skipped_tests (bool): Whether to ignore builds which skipped
	testing.
	score_threshold (float): Count a finding as part of the aggregate if
	its score meets this threshold.
	build_frequency_threshold (float): Record an aggregate finding if
	the frequency of findings meets this threshold, i.e. a 0.5
	threshold means that a finding will be recorded if
	num_findings / num_inspected_builds >= 0.5.
	"""
	assert self.m.buildbucket_util.is_tryjob, "invalid command in non-try"
	with self.m.step.nest(step_name):
	args = [
	"check-try",
	"-builder",
	builder or self.m.buildbucket_util.full_builder_name(),
	"-change-num",
	change_num or self.m.buildbucket.build.input.gerrit_changes[0].change,
	"-build-status",
	self.m.buildbucket_util.build_status(exception),
	"-summary-markdown-path",
	self._summary_markdown_path(summary_markdown),
	"-scrub-header",
	AUTOCORRELATOR_HEADER,
	"-scrub-footer",
	AUTOCORRELATOR_FOOTER,
	"-json-output",
	self.m.json.output(),
	]
	if ignore_failed_build:
	args.append("-ignore-failed-build")
	if ignore_skipped_build:
	args.append("-ignore-skipped-build")
	if ignore_skipped_tests:
	args.append("-ignore-skipped-tests")
	step = self("run autocorrelator", args)
	findings = step.json.output
	if not findings:
	return step
	findings_meeting_threshold = []
	for finding in findings:
	if finding["score"] >= score_threshold:
	findings_meeting_threshold.append(finding)
	try_build_link = self.m.buildbucket.build_url(
	build_id=finding["build_id"]
	)
	step.presentation.links[finding["build_id"]] = try_build_link
	build_frequency = len(findings_meeting_threshold) / float(len(findings))
	if build_frequency >= build_frequency_threshold:
	self.findings.append(
	"Similar failures found in try: %0.2f avg similarity in %s\n\n%s"
	% (
	sum(
	(finding["score"] for finding in findings_meeting_threshold)
	)
	/ len(findings_meeting_threshold),
	pluralize("build", len(findings_meeting_threshold)),
	"\n".join(
	"- %s" % build_link
	for build_link in step.presentation.links.itervalues()
	),
	),
	)
	return step

	def set_properties(self, presentation):
	"""Set output properties for current autocorrelator findings."""
	presentation.properties["num_autocorrelator_findings"] = len(self.findings)

	def compose_exception(self, original_exc):
	"""Compose a modified StepFailure with the current autocorrelator
	findings.

	Args:
	original_exc (Exception): Exception to modify.

	Returns:
	StepFailure: containing the modified summary markdown
	including autocorrelator findings.
	"""
	# If the exception is not a StepFailure then there's no consistent way
	# to obtain the original summary markdown. Likewise, if there are no
	# findings then there's no context to add to the exception.
	if not isinstance(original_exc, self.m.step.StepFailure) or not self.findings:
	return original_exc

	# Construct a summary markdown for current findings.
	autocorrelator_summary = self.m.buildbucket_util.summary_message(
	"\n\n".join(self.findings),
	"findings truncated, see autocorrelator steps for full details",
	truncate_length=(
	AUTOCORRELATOR_MAX_SUMMARY_SIZE
	# Include space for header and footer text.
	- len(AUTOCORRELATOR_HEADER)
	- len(AUTOCORRELATOR_FOOTER)
	# Include space for newline characters between each component.
	# See join() below.
	- 2 * 3
	),
	escape_markdown=False,
	)
	# Stitch together the full summary markdown.
	summary_markdown = "\n\n".join(
	(
	AUTOCORRELATOR_HEADER,
	autocorrelator_summary,
	AUTOCORRELATOR_FOOTER,
	original_exc.reason,
	)
	)

	if isinstance(original_exc, self.m.step.InfraFailure):
	return self.m.step.InfraFailure(summary_markdown)
	elif isinstance(original_exc, self.m.step.StepWarning):
	return self.m.step.StepWarning(summary_markdown)
	return self.m.step.StepFailure(summary_markdown)

	def __call__(self, step_name, args):
	return self.m.step(
	step_name, [self._autocorrelator_tool] + args, infra_step=True
	)

	def _summary_markdown_path(self, summary_markdown):
	"""Write summary markdown to a file and return the Path."""
	path = self.m.path.mkstemp("summary_md")
	self.m.file.write_text("write summary markdown", path, str(summary_markdown))
	return path

	@property
	def _autocorrelator_tool(self):
	return self.m.cipd.ensure_tool(AUTOCORRELATOR_PATH, AUTOCORRELATOR_VERSION)