cmd/autocorrelator/check_try.go

// Copyright 2021 The Fuchsia Authors.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"

	"github.com/maruel/subcommands"
	"github.com/texttheater/golang-levenshtein/levenshtein"
	"go.chromium.org/luci/auth"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/luciexe/exe"
	"google.golang.org/genproto/protobuf/field_mask"

	"go.fuchsia.dev/infra/buildbucket"
	"go.fuchsia.dev/infra/cmd/autocorrelator/compare"
	"go.fuchsia.dev/infra/cmd/autocorrelator/findings"
	"go.fuchsia.dev/infra/cmd/autocorrelator/scrub"
)

func cmdCheckTry(authOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "check-try -builder <project/bucket/builder> -change-num <change-num> -build-status <build-status-code> -search-range <search-range> -summary-markdown-path <path/to/summary/markdown> -json-output <json-output>",
		ShortDesc: "Compare text similarity between a summary markdown and recent try builds' summary markdowns.",
		LongDesc:  "Compare text similarity between the provided -summary-markdown and the summary markdown of -search-range recent builds of a try -builder.",
		CommandRun: func() subcommands.CommandRun {
			c := &checkTryRun{}
			c.Init(authOpts)
			return c
		},
	}
}

type checkTryRun struct {
	commonFlags
	// TODO(atyfto): Fix the buildbucket library to return a BuilderID instead
	// of a flag.
	builder             buildbucket.BuilderIDFlag
	changeNum           int64
	buildStatus         int
	searchRange         int
	summaryMarkdownPath string
	ignoreSkippedBuild  bool
	ignoreSkippedTests  bool
	ignoreFailedBuild   bool
	scrubHeader         string
	scrubFooter         string
	jsonOutput          string
}

func (c *checkTryRun) Init(defaultAuthOpts auth.Options) {
	c.commonFlags.Init(defaultAuthOpts)
	c.Flags.Var(&c.builder, "builder", "Fully-qualified Buildbucket try builder name to inspect.")
	c.Flags.Int64Var(&c.changeNum, "change-num", 0, "Change number to filter out.")
	c.Flags.IntVar(&c.buildStatus, "build-status", 20, "Build status to filter on. Defaults to FAILURE.")
	c.Flags.IntVar(&c.searchRange, "search-range", 10, "Inspect up to this many recent builds.")
	c.Flags.StringVar(&c.summaryMarkdownPath, "summary-markdown-path", "", "Path to summary markdown input file.")
	c.Flags.BoolVar(&c.ignoreSkippedBuild, "ignore-skipped-build", false, "Whether to ignore builds with unaffected build graphs.")
	c.Flags.BoolVar(&c.ignoreSkippedTests, "ignore-skipped-tests", false, "Whether to ignore builds which skipped testing.")
	c.Flags.BoolVar(&c.ignoreFailedBuild, "ignore-failed-build", false, "Whether to ignore builds which failed to build.")
	c.Flags.StringVar(&c.scrubHeader, "scrub-header", "", "Header of section(s) to scrub out.")
	c.Flags.StringVar(&c.scrubFooter, "scrub-footer", "", "Footer of section(s) to scrub out.")
	c.Flags.StringVar(&c.jsonOutput, "json-output", "-", "Path to output finding to.")
}

func (c *checkTryRun) Parse() error {
	if err := c.commonFlags.Parse(); err != nil {
		return err
	}
	if &c.builder == nil {
		return errors.New("-builder is required")
	}
	if c.changeNum == 0 {
		return errors.New("-change-num is required")
	}
	if c.summaryMarkdownPath == "" {
		return errors.New("-summary-markdown-path is required")
	}
	if c.scrubHeader == "" {
		return errors.New("-scrub-header is required")
	}
	if c.scrubFooter == "" {
		return errors.New("-scrub-footer is required")
	}
	if c.jsonOutput == "" {
		return errors.New("-json-output is required")
	}
	return nil
}

// checkTry is the main algorithm which backs this subcommand.
//
// Given a list of builds, return a list of summary similarity findings.
// Unlike the CI checker algorithm, this function does not assume any order in
// the list of builds, and does not depend on a git log.
//
// An output with many high similarity scores could indicate systemic failure.
//
// The `comparator` argument is a text-based comparator that returns a
// similarity score between the input summary markdown and a build's summary
// markdown.
func checkTry(builds []*buildbucketpb.Build, changeNum int64, status buildbucketpb.Status, summaryMarkdown string, comparator compare.TextComparator, ignoreSkippedBuild, ignoreSkippedTests, ignoreFailedBuild bool, scrubHeader, scrubFooter string) []*findings.SummarySimilarity {
	var fs []*findings.SummarySimilarity
	for _, build := range builds {
		// Unfortunately we cannot exclude canceled builds in the query itself,
		// so we must filter them here.
		if build.Status == buildbucketpb.Status_CANCELED {
			continue
		}
		changes := build.Input.GerritChanges
		// A build with no input changes was likely manually triggered using `bb
		// add`. It may have been triggered on an old commit, in which case it
		// won't be representative of the builder's current status, so we should
		// skip it.
		if len(changes) == 0 {
			continue
		}
		// If we find a build with an identical change number, skip it.
		if changeNum == changes[0].Change {
			continue
		}
		// If we find a build with no output properties, it generally cannot be
		// further analyzed.
		if build.Output.Properties == nil {
			continue
		}
		// Conditionally ignore builds with unaffected build graph, e.g. when
		// checking for build failures.
		if ignoreSkippedBuild {
			var skipped bool
			exe.ParseProperties(build.Output.Properties, map[string]any{
				"skipped_because_unaffected": &skipped,
			})
			if skipped {
				continue
			}
		}
		// Conditionally ignore builds which skipped testing, e.g. when checking
		// for test failures.
		if ignoreSkippedTests {
			var noWork bool
			exe.ParseProperties(build.Output.Properties, map[string]any{
				"affected_tests_no_work": &noWork,
			})
			if noWork {
				continue
			}
		}
		// Conditionally ignore builds which failed to build, e.g. when checking
		// for test failures.
		if ignoreFailedBuild {
			var buildFailed bool
			exe.ParseProperties(build.Output.Properties, map[string]any{
				"failed_to_build": &buildFailed,
			})
			if buildFailed {
				continue
			}
		}
		// If we encounter a green build, add the finding: the existence of a
		// green build should reduce the confidence we have in "try" being
		// systemically busted.
		if build.Status == buildbucketpb.Status_SUCCESS {
			fs = append(fs, &findings.SummarySimilarity{
				BuildId: strconv.FormatInt(build.Id, 10),
				Score:   0.0,
				IsGreen: true,
			})
			// If we encounter a build with a matching status, compute the
			// summary markdown similarity.
		} else if build.Status == status {
			fs = append(fs, &findings.SummarySimilarity{
				BuildId: strconv.FormatInt(build.Id, 10),
				Score:   comparator.Compare(scrub.ScrubSection(build.SummaryMarkdown, scrubHeader, scrubFooter), summaryMarkdown),
			})
		}
	}
	return fs
}

func (c *checkTryRun) main() error {
	ctx := context.Background()
	authClient, err := auth.NewAuthenticator(ctx, auth.OptionalLogin, c.parsedAuthOpts).Client()
	if err != nil {
		return fmt.Errorf("failed to initialize auth client: %w", err)
	}

	// Fetch the last -search-range completed builds of -builder.
	buildsClient := buildbucketpb.NewBuildsPRPCClient(&prpc.Client{
		C:    authClient,
		Host: c.bbHost,
	})
	// TODO(atyfto): Fix the `buildbucket` library so we can get rid of this
	// type assertion.
	builder, ok := c.builder.Get().(*buildbucketpb.BuilderID)
	if !ok {
		return errors.New("builder input is invalid")
	}
	resp, err := buildsClient.SearchBuilds(ctx, &buildbucketpb.SearchBuildsRequest{
		Predicate: &buildbucketpb.BuildPredicate{
			Builder: builder,
			Status:  buildbucketpb.Status_ENDED_MASK,
		},
		Fields: &field_mask.FieldMask{Paths: []string{
			"builds.*.id",
			"builds.*.input.gerrit_changes",
			"builds.*.output.properties",
			"builds.*.status",
			"builds.*.summary_markdown",
		}},
		PageSize: int32(c.searchRange),
	})
	if err != nil {
		return fmt.Errorf("failed to query builds for %s: %w", c.builder.String(), err)
	}
	builds := resp.Builds
	// Run the main algorithm.
	summaryMarkdown, err := os.ReadFile(c.summaryMarkdownPath)
	if err != nil {
		return fmt.Errorf("could not read summary markdown input: %w", err)
	}
	fs := checkTry(builds, c.changeNum, buildbucketpb.Status(c.buildStatus), string(summaryMarkdown), compare.LevenshteinComparator{Opts: levenshtein.DefaultOptions}, c.ignoreSkippedBuild, c.ignoreSkippedTests, c.ignoreFailedBuild, c.scrubHeader, c.scrubFooter)

	// Emit summary similarity findings to -json-output.
	out := os.Stdout
	if c.jsonOutput != "-" {
		out, err = os.Create(c.jsonOutput)
		if err != nil {
			return err
		}
		defer out.Close()
	}
	if err := json.NewEncoder(out).Encode(fs); err != nil {
		return fmt.Errorf("failed to encode: %w", err)
	}
	return nil
}

func (c *checkTryRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	if err := c.Parse(); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}

	if err := c.main(); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	return 0
}