cmd/cl-util/run_postsubmit_tryjobs.go - infra/infra - Git at Google

 // Copyright 2022 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 package main

 import (
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log"
 	"net/url"
 	"os"
 	"sort"
 	"strings"

 	"github.com/golang/protobuf/proto"
 	"github.com/maruel/subcommands"
 	"go.chromium.org/luci/auth"
 	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
 	gerritpb "go.chromium.org/luci/common/proto/gerrit"
 	cvpb "go.chromium.org/luci/cv/api/config/v2"
 	"golang.org/x/exp/slices"

 	"go.fuchsia.dev/infra/buildbucket"
 	"go.fuchsia.dev/infra/flagutil"
 	"go.fuchsia.dev/infra/gerrit"
 	"go.fuchsia.dev/infra/gitiles"
 	"google.golang.org/genproto/protobuf/field_mask"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/types/known/structpb"
 )

 const rptLongDesc = `
 run-postsubmit-tryjobs run all available presubmit tryjobs for a CL that are
 required by postsubmit. Due to a number of reasons
 outlined in go/what-belongs-in-presubmit we can't run all tryjobs on all
 changes by default, so this tool is for allowing automated tooling or humans
 to specify which changes to run all builders for.
 `

 const buildbucketHost = "cr-buildbucket.appspot.com"

 func cmdRunPostsubmitTryjobs(authOpts auth.Options) *subcommands.Command {
 	return &subcommands.Command{
 		UsageLine: "run-postsubmit-tryjobs <CL URL>",
 		ShortDesc: "Trigger available tryjobs that are required for postsubmit for a CL.",
 		LongDesc:  rptLongDesc,
 		CommandRun: func() subcommands.CommandRun {
 			c := &rptCmd{}
 			c.Init(authOpts)
 			return c
 		},
 	}
 }

 type rptCmd struct {
 	commonFlags
 	gerritChangeID        int64
 	gerritPatchset        int64
 	jsonOutputFile        string
 	CommitQueueCfgPath    string
 	LUCIConfigHost        string
 	LUCIConfigPath        string
 	LUCIConfigProject     string
 	CrBuildBucketCfgPaths flagutil.RepeatedStringValue
 	dryRun                bool
 	force                 bool
 	allPresubmit          bool
 	verbose               bool
 }

 type BuildersToTrigger struct {
 	Builders []string `json:"builders"`
 }

 type LUCIConfigurationFiles struct {
 	commitQueueConfig    *cvpb.Config
 	crBuildBucketConfigs map[string]*buildbucketpb.BuildbucketCfg
 }

 func (c *rptCmd) Init(defaultAuthOpts auth.Options) {
 	c.commonFlags.Init(defaultAuthOpts)
 	c.Flags.Int64Var(
 		&c.gerritChangeID,
 		"gerrit-change-id",
 		0,
 		"The ChangeID of the GerritChange to trigger builders against.",
 	)
 	c.Flags.StringVar(
 		&c.LUCIConfigHost,
 		"luci-config-host",
 		"turquoise-internal.googlesource.com",
 		"Gerrit host for the project containing the LUCI configuration files.",
 	)
 	c.Flags.StringVar(
 		&c.LUCIConfigProject,
 		"luci-config-project",
 		"integration",
 		"Project name containing the LUCI configuration files.",
 	)
 	c.Flags.StringVar(
 		&c.CommitQueueCfgPath,
 		"commit-queue-cfg-path",
 		"infra/config/generated/turquoise/luci/commit-queue.cfg",
 		"Path to the LUCI commit-queue.cfg configuration file.",
 	)
 	c.Flags.Var(
 		&c.CrBuildBucketCfgPaths,
 		"cr-buildbucket-cfg-project-path",
 		"One or more comma-separated strings containing a project name and path to a LUCI cr-buildbucket.cfg configuration file. e.g. 'project,path/to/cr-buildbucket.cfg'",
 	)
 	c.Flags.BoolVar(
 		&c.allPresubmit,
 		"trigger-all-presubmit",
 		false,
 		"Whether to include presubmit builders not tagged for run-postsubmit-tryjobs. Projects outside Fuchsia that have not tagged their builders should enable this flag.",
 	)
 	c.Flags.BoolVar(
 		&c.force,
 		"f",
 		false,
 		"Whether to skip command line confirmation when triggering builders.",
 	)
 	c.Flags.BoolVar(
 		&c.verbose,
 		"v",
 		false,
 		"Whether to print all builders to be run.",
 	)
 	c.Flags.StringVar(
 		&c.jsonOutputFile,
 		"json-output",
 		"",
 		"Filepath to write json output to. Use '-' for stdout.",
 	)
 	c.Flags.BoolVar(
 		&c.dryRun,
 		"dry-run",
 		false,
 		"Whether to actually trigger the builders or just print out which builders would have been triggered.",
 	)
 }

 func (c *rptCmd) Parse(a subcommands.Application, args []string) error {
 	if len(args) != 0 && !strings.HasPrefix(args[0], "-") {
 		// The first positional argument is the changelink, parse it into constituent parts.
 		changeUrl, err := url.Parse(args[0])
 		if err != nil {
 			return err
 		}

 		gerritHost, changeID, err := gerrit.ResolveChangeUrl(changeUrl)
 		if err != nil {
 			return err
 		}
 		c.gerritHost = gerritHost
 		c.gerritChangeID = changeID
 	}

 	// Set placeholder value for gerritProject if it isn't set at this point.
 	// commonFlags.Parse includes validation for gerritProject to not be null,
 	// but we set it in getChangeDetails.
 	if c.gerritProject == "" {
 		c.gerritProject = "placeholder-project-name"
 	}

 	// Custom vars can't have defaults set so populate here in case it's unset.
 	if len(c.CrBuildBucketCfgPaths) == 0 {
 		c.CrBuildBucketCfgPaths = flagutil.RepeatedStringValue{
 			"turquoise,infra/config/generated/turquoise/luci/cr-buildbucket.cfg",
 			"fuchsia,infra/config/generated/fuchsia/luci/cr-buildbucket.cfg",
 		}
 	}

 	if err := c.commonFlags.Parse(); err != nil {
 		return err
 	}

 	return nil
 }

 func (c *rptCmd) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
 	if err := c.Parse(a, args); err != nil {
 		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
 		return 1
 	}

 	if err := c.main(); err != nil {
 		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
 		return 1
 	}
 	return 0
 }

 func (c *rptCmd) main() error {
 	if !c.verbose {
 		log.SetOutput(io.Discard)
 	}

 	ctx := context.Background()

 	buildClient, err := buildbucket.NewBuildsClient(
 		ctx,
 		buildbucketHost,
 		c.commonFlags.parsedAuthOpts,
 	)
 	if err != nil {
 		return err
 	}

 	authClient, err := auth.NewAuthenticator(ctx, auth.OptionalLogin, c.commonFlags.parsedAuthOpts).Client()
 	if err != nil {
 		return fmt.Errorf("failed to get authenticated http client: %w", err)
 	}

 	gitilesClient, err := gitiles.NewClient(c.LUCIConfigHost, c.LUCIConfigProject, authClient)
 	if err != nil {
 		return err
 	}

 	lucicfg, err := c.getLUCIConfigs(ctx, *gitilesClient)
 	if err != nil {
 		return err
 	}

 	gerritClient, err := gerrit.NewClient(c.gerritHost, c.gerritProject, authClient)
 	if err != nil {
 		return err
 	}

 	if err = c.getChangeDetails(ctx, *gerritClient); err != nil {
 		return err
 	}

 	availablePresubmitBuilders := c.getAvailablePresubmitBuilders(lucicfg, c.gerritHost, c.gerritProject)

 	triggeredBuilds, err := c.getTriggeredBuildsForChange(ctx, buildClient)
 	if err != nil {
 		return err
 	}

 	triggeredBuilds = c.filterFailedBuilders(triggeredBuilds)
 	missingBuilders := c.getMissingBuilders(availablePresubmitBuilders, triggeredBuilds)
 	return c.triggerMissingBuilders(ctx, buildClient, missingBuilders)
 }

 // getLUCIConfigs fetches the LUCI configuration files contained at
 // the project specified in c.LUCIConfigProject and c.LUCIConfigPath
 func (c *rptCmd) getLUCIConfigs(ctx context.Context, gitilesClient gitiles.Client) (LUCIConfigurationFiles, error) {
 	log.Println("Downloading LUCI configuration files...")
 	// Set non-nil values for the configs.
 	commitQueueConfig := &cvpb.Config{}
 	crBuildBucketConfigs := map[string]*buildbucketpb.BuildbucketCfg{}

 	commitQueueFileContents, err := gitilesClient.DownloadFile(
 		ctx,
 		c.CommitQueueCfgPath,
 		"refs/heads/main",
 	)
 	if err != nil {
 		return LUCIConfigurationFiles{}, err
 	}

 	if err := proto.UnmarshalText(commitQueueFileContents, commitQueueConfig); err != nil {
 		return LUCIConfigurationFiles{}, err
 	}

 	for _, projectPath := range c.CrBuildBucketCfgPaths {
 		projectPathSplit := strings.Split(projectPath, ",")
 		if len(projectPathSplit) != 2 {
 			return LUCIConfigurationFiles{}, errors.New(
 				fmt.Sprintf(
 					"Invalid entry to cr-buildbucket-cfg-project-path, expected 'project,path/to/cr-buildbucket.cfg' got %s",
 					projectPath,
 				),
 			)
 		}
 		project := projectPathSplit[0]
 		path := projectPathSplit[1]

 		cfg := &buildbucketpb.BuildbucketCfg{}
 		CrBuildBucketFileContents, err := gitilesClient.DownloadFile(
 			ctx,
 			path,
 			"refs/heads/main",
 		)
 		if err != nil {
 			return LUCIConfigurationFiles{}, err
 		}

 		if err = proto.UnmarshalText(CrBuildBucketFileContents, cfg); err != nil {
 			return LUCIConfigurationFiles{}, err
 		}

 		crBuildBucketConfigs[project] = cfg
 	}

 	return LUCIConfigurationFiles{
 		commitQueueConfig:    commitQueueConfig,
 		crBuildBucketConfigs: crBuildBucketConfigs,
 	}, nil
 }

 // getChangeDetails queries Gerrit for the target change to determine
 // the ChangeRef, ChangeRepo, Project and Patchset (if not specified)
 func (c *rptCmd) getChangeDetails(ctx context.Context, gerritClient gerrit.Client) error {
 	change, err := gerritClient.GetChange(
 		ctx,
 		c.gerritChangeID,
 		gerritpb.QueryOption_ALL_REVISIONS,
 	)
 	if err != nil {
 		return err
 	}

 	c.gerritPatchset = int64(change.Revisions[change.CurrentRevision].Number)
 	c.gerritProject = change.Project

 	return nil
 }

 // getAvailablePresubmitBuilders parses through available builders in
 // presubmit and if an equivalent builder is required in postsubmit, adds it to
 // the list of builders we check to trigger.
 func (c *rptCmd) getAvailablePresubmitBuilders(lucicfg LUCIConfigurationFiles, gerritHost string, gerritProject string) []*buildbucketpb.BuilderID {
 	// Throw buckets into a map for cross referencing.
 	flaggedBuildersByBucketAndProjectMap := map[string]map[string]map[string]bool{}
 	for project, cfg := range lucicfg.crBuildBucketConfigs {
 		if _, ok := flaggedBuildersByBucketAndProjectMap[project]; !ok {
 			flaggedBuildersByBucketAndProjectMap[project] = map[string]map[string]bool{}
 		}
 		for _, bucket := range cfg.GetBuckets() {
 			flaggedBuildersByBucketAndProjectMap[project][bucket.GetName()] = getBuildersFlaggedForRpt(bucket)
 		}
 	}

 	// Find CQ for the target change.
 	configGroups := lucicfg.commitQueueConfig.GetConfigGroups()
 	var availablePresubmitBuilders []*buildbucketpb.BuilderID

 	for _, cfg := range configGroups {
 		projectMatch := cfg.GetGerrit()[0].GetProjects()[0].GetName() == gerritProject
 		cfgUrl, _ := url.Parse(cfg.GetGerrit()[0].GetUrl())
 		hostMatch := cfgUrl.Host == gerritHost
 		if projectMatch && hostMatch {
 			for _, builder := range cfg.GetVerifiers().GetTryjob().GetBuilders() {
 				builderSplit := strings.Split(builder.GetName(), "/")
 				builderID := &buildbucketpb.BuilderID{
 					Project: builderSplit[0],
 					Bucket:  builderSplit[1],
 					Builder: builderSplit[2],
 				}

 				if c.allPresubmit || flaggedBuildersByBucketAndProjectMap[builderID.Project][builderID.Bucket][builderID.Builder] {
 					availablePresubmitBuilders = append(availablePresubmitBuilders, builderID)
 				}
 			}
 		}
 	}

 	// Sort for logging output clarity.
 	sort.SliceStable(availablePresubmitBuilders, func(i, j int) bool {
 		return availablePresubmitBuilders[i].Builder < availablePresubmitBuilders[j].Builder
 	})

 	return availablePresubmitBuilders
 }

 // getTriggeredBuildersForChange calls the buildbucket SearchBuilds RPC
 // and reports back all builders that have already triggered against the change.
 func (c *rptCmd) getTriggeredBuildsForChange(ctx context.Context, buildClient buildbucketpb.BuildsClient) ([]*buildbucketpb.Build, error) {
 	// Pull all builds triggered against the CL.
 	resp, err := buildClient.SearchBuilds(
 		ctx,
 		&buildbucketpb.SearchBuildsRequest{
 			Predicate: &buildbucketpb.BuildPredicate{
 				GerritChanges: []*buildbucketpb.GerritChange{
 					{
 						Host:     c.gerritHost,
 						Project:  c.gerritProject,
 						Change:   c.gerritChangeID,
 						Patchset: c.gerritPatchset,
 					},
 				},
 			},
 			Fields: &field_mask.FieldMask{Paths: []string{
 				"builds.*.builder",
 				"builds.*.status",
 			}},
 			PageSize: 1000,
 		},
 	)
 	if err != nil {
 		return []*buildbucketpb.Build{}, err
 	}

 	return resp.GetBuilds(), nil
 }

 // filterFailedBuilders filters failed builders out of the triggered builders
 // so that they will be re-triggered by run-postsubmit-tryjobs.
 func (c *rptCmd) filterFailedBuilders(triggeredBuilds []*buildbucketpb.Build) []*buildbucketpb.Build {
 	var filteredBuilds []*buildbucketpb.Build
 	for _, build := range triggeredBuilds {
 		if !slices.Contains(
 			[]buildbucketpb.Status{
 				buildbucketpb.Status_FAILURE,
 				buildbucketpb.Status_INFRA_FAILURE,
 			},
 			build.GetStatus(),
 		) {
 			filteredBuilds = append(filteredBuilds, build)
 		}
 	}
 	return filteredBuilds
 }

 // getMissingBuilders finds the diff between availablePresubmitBuilders and the
 // builds already triggered by the change.
 func (c *rptCmd) getMissingBuilders(availablePresubmitBuilders []*buildbucketpb.BuilderID, triggeredPresubmitBuilds []*buildbucketpb.Build) []*buildbucketpb.BuilderID {
 	var missingBuilders []*buildbucketpb.BuilderID
 	var triggeredBuilders []string
 	for _, build := range triggeredPresubmitBuilds {
 		triggeredBuilders = append(triggeredBuilders, build.GetBuilder().GetBuilder())
 	}

 	// Find which builders haven't been triggered yet.
 	for _, builder := range availablePresubmitBuilders {
 		if !slices.Contains(triggeredBuilders, builder.GetBuilder()) && !builderSliceContains(missingBuilders, builder) {
 			missingBuilders = append(missingBuilders, builder)
 		}
 	}

 	return missingBuilders
 }

 // triggerMissingBuilders triggers the diff between all builders available
 // and the builders that have already been triggered for the change.
 func (c *rptCmd) triggerMissingBuilders(ctx context.Context, buildClient buildbucketpb.BuildsClient, missingBuilders []*buildbucketpb.BuilderID) error {
 	// If we have no builders to trigger, return.
 	if len(missingBuilders) == 0 {
 		log.Println("No builders found to trigger.")
 		return nil
 	}

 	// Print out list of builders to be triggered for logging.
 	for _, builder := range missingBuilders {
 		log.Printf("Builder: %s\n", builder.GetBuilder())
 	}

 	// Require command line confirmation.  Automated tools should pass the -f flag.
 	if !(c.force || c.dryRun) {
 		// The cost has been calculated roughly on average as of 2022 to be $0.57 per builder triggered
 		// GCE Cost: $0.53/h
 		// Orchestrator builder: 0.45h
 		// Subbuild builder: 0.45h
 		// Testing Subtasks: 0.17h
 		fmt.Printf("You are about to trigger %d tryjobs with an estimated cost of $%.2f USD. Additionally, this will likely consume inelastic resources. Do you wish to proceed? (y/n)\n", len(missingBuilders), 0.57*float64(len(missingBuilders)))
 		confirm := ""
 		fmt.Scanln(&confirm)
 		if !slices.Contains([]string{"y", "Y", "yes", "Yes", "YES"}, confirm) {
 			fmt.Println("Process aborted.")
 			return nil
 		} else {
 			fmt.Println("You can skip this confirmation next time by passing the -f flag.")
 		}
 	}

 	// Assemble requests into a batch.
 	var requestBatch []*buildbucketpb.BatchRequest_Request
 	var triggeredBuilders []string
 	for _, builder := range missingBuilders {
 		req := &buildbucketpb.BatchRequest_Request{
 			Request: &buildbucketpb.BatchRequest_Request_ScheduleBuild{
 				ScheduleBuild: &buildbucketpb.ScheduleBuildRequest{
 					Builder: builder,
 					GerritChanges: []*buildbucketpb.GerritChange{
 						{
 							Host:     c.gerritHost,
 							Project:  c.gerritProject,
 							Change:   c.gerritChangeID,
 							Patchset: c.gerritPatchset,
 						},
 					},
 					// Slightly below default priority.
 					Priority: 31,
 				},
 			},
 		}
 		triggeredBuilders = append(triggeredBuilders, builder.Builder)
 		requestBatch = append(requestBatch, req)

 	}

 	if !c.dryRun {
 		resp, err := buildClient.Batch(
 			ctx,
 			&buildbucketpb.BatchRequest{
 				Requests: requestBatch,
 			},
 		)
 		if err != nil {
 			return err
 		}

 		jsonResp, err := json.MarshalIndent(resp, "", "  ")
 		if err != nil {
 			return err
 		}

 		log.Printf(
 			"BatchRequest response: \n%s",
 			jsonResp,
 		)
 	}

 	if c.jsonOutputFile != "" {
 		// Assemble builder data into easy to consume json output.
 		jsonOutput := &BuildersToTrigger{
 			Builders: triggeredBuilders,
 		}

 		// Marshal output to json.
 		var rawJSON []byte
 		var err error
 		var f io.WriteCloser = os.Stdout
 		if c.jsonOutputFile != "-" {
 			if f, err = os.Create(c.jsonOutputFile); err != nil {
 				return err
 			}
 			defer f.Close()
 		}

 		if c.jsonOutputFile == "-" {
 			rawJSON, err = json.MarshalIndent(jsonOutput, "", "    ")
 			rawJSON = append(rawJSON, '\n')
 		} else {
 			rawJSON, err = json.Marshal(jsonOutput)
 			rawJSON = append(rawJSON, '\n')
 		}

 		if err != nil {
 			return err
 		}

 		if _, err := f.Write(rawJSON); err != nil {
 			return err
 		}
 	}

 	return nil
 }

 // slices.Contains doesn't really work with slices of pointers, so this helper
 // performs a low effort check for inclusion.
 func builderSliceContains(slice []*buildbucketpb.BuilderID, builder *buildbucketpb.BuilderID) bool {
 	for _, b := range slice {
 		if b.GetBuilder() == builder.GetBuilder() {
 			return true
 		}
 	}
 	return false
 }

 // getBuildersFlaggedForRpt finds builders with the `run_postsubmit_tryjobs_include` property.
 func getBuildersFlaggedForRpt(bucket *buildbucketpb.Bucket) map[string]bool {
 	flaggedBuilders := map[string]bool{}
 	for _, b := range bucket.GetSwarming().GetBuilders() {
 		properties := &structpb.Struct{}
 		protojson.Unmarshal([]byte(b.GetProperties()), properties)
 		if builderTags, ok := properties.GetFields()["$fuchsia/builder_tags"]; ok {
 			if runPostsubmitTryjobsInclude, ok := builderTags.GetStructValue().GetFields()["run_postsubmit_tryjobs_include"]; ok {
 				flaggedBuilders[b.GetName()] = runPostsubmitTryjobsInclude.GetBoolValue()
 			} else {
 				flaggedBuilders[b.GetName()] = false
 			}
 		} else {
 			flaggedBuilders[b.GetName()] = false
 		}
 	}

 	return flaggedBuilders
 }
	// Copyright 2022 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	package main

	import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"sort"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/maruel/subcommands"
	"go.chromium.org/luci/auth"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	gerritpb "go.chromium.org/luci/common/proto/gerrit"
	cvpb "go.chromium.org/luci/cv/api/config/v2"
	"golang.org/x/exp/slices"

	"go.fuchsia.dev/infra/buildbucket"
	"go.fuchsia.dev/infra/flagutil"
	"go.fuchsia.dev/infra/gerrit"
	"go.fuchsia.dev/infra/gitiles"
	"google.golang.org/genproto/protobuf/field_mask"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/types/known/structpb"
	)

	const rptLongDesc = `
	run-postsubmit-tryjobs run all available presubmit tryjobs for a CL that are
	required by postsubmit. Due to a number of reasons
	outlined in go/what-belongs-in-presubmit we can't run all tryjobs on all
	changes by default, so this tool is for allowing automated tooling or humans
	to specify which changes to run all builders for.
	`

	const buildbucketHost = "cr-buildbucket.appspot.com"

	func cmdRunPostsubmitTryjobs(authOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
	UsageLine: "run-postsubmit-tryjobs <CL URL>",
	ShortDesc: "Trigger available tryjobs that are required for postsubmit for a CL.",
	LongDesc: rptLongDesc,
	CommandRun: func() subcommands.CommandRun {
	c := &rptCmd{}
	c.Init(authOpts)
	return c
	},
	}
	}

	type rptCmd struct {
	commonFlags
	gerritChangeID int64
	gerritPatchset int64
	jsonOutputFile string
	CommitQueueCfgPath string
	LUCIConfigHost string
	LUCIConfigPath string
	LUCIConfigProject string
	CrBuildBucketCfgPaths flagutil.RepeatedStringValue
	dryRun bool
	force bool
	allPresubmit bool
	verbose bool
	}

	type BuildersToTrigger struct {
	Builders []string `json:"builders"`
	}

	type LUCIConfigurationFiles struct {
	commitQueueConfig *cvpb.Config
	crBuildBucketConfigs map[string]*buildbucketpb.BuildbucketCfg
	}

	func (c *rptCmd) Init(defaultAuthOpts auth.Options) {
	c.commonFlags.Init(defaultAuthOpts)
	c.Flags.Int64Var(
	&c.gerritChangeID,
	"gerrit-change-id",
	0,
	"The ChangeID of the GerritChange to trigger builders against.",
	)
	c.Flags.StringVar(
	&c.LUCIConfigHost,
	"luci-config-host",
	"turquoise-internal.googlesource.com",
	"Gerrit host for the project containing the LUCI configuration files.",
	)
	c.Flags.StringVar(
	&c.LUCIConfigProject,
	"luci-config-project",
	"integration",
	"Project name containing the LUCI configuration files.",
	)
	c.Flags.StringVar(
	&c.CommitQueueCfgPath,
	"commit-queue-cfg-path",
	"infra/config/generated/turquoise/luci/commit-queue.cfg",
	"Path to the LUCI commit-queue.cfg configuration file.",
	)
	c.Flags.Var(
	&c.CrBuildBucketCfgPaths,
	"cr-buildbucket-cfg-project-path",
	"One or more comma-separated strings containing a project name and path to a LUCI cr-buildbucket.cfg configuration file. e.g. 'project,path/to/cr-buildbucket.cfg'",
	)
	c.Flags.BoolVar(
	&c.allPresubmit,
	"trigger-all-presubmit",
	false,
	"Whether to include presubmit builders not tagged for run-postsubmit-tryjobs. Projects outside Fuchsia that have not tagged their builders should enable this flag.",
	)
	c.Flags.BoolVar(
	&c.force,
	"f",
	false,
	"Whether to skip command line confirmation when triggering builders.",
	)
	c.Flags.BoolVar(
	&c.verbose,
	"v",
	false,
	"Whether to print all builders to be run.",
	)
	c.Flags.StringVar(
	&c.jsonOutputFile,
	"json-output",
	"",
	"Filepath to write json output to. Use '-' for stdout.",
	)
	c.Flags.BoolVar(
	&c.dryRun,
	"dry-run",
	false,
	"Whether to actually trigger the builders or just print out which builders would have been triggered.",
	)
	}

	func (c *rptCmd) Parse(a subcommands.Application, args []string) error {
	if len(args) != 0 && !strings.HasPrefix(args[0], "-") {
	// The first positional argument is the changelink, parse it into constituent parts.
	changeUrl, err := url.Parse(args[0])
	if err != nil {
	return err
	}

	gerritHost, changeID, err := gerrit.ResolveChangeUrl(changeUrl)
	if err != nil {
	return err
	}
	c.gerritHost = gerritHost
	c.gerritChangeID = changeID
	}

	// Set placeholder value for gerritProject if it isn't set at this point.
	// commonFlags.Parse includes validation for gerritProject to not be null,
	// but we set it in getChangeDetails.
	if c.gerritProject == "" {
	c.gerritProject = "placeholder-project-name"
	}

	// Custom vars can't have defaults set so populate here in case it's unset.
	if len(c.CrBuildBucketCfgPaths) == 0 {
	c.CrBuildBucketCfgPaths = flagutil.RepeatedStringValue{
	"turquoise,infra/config/generated/turquoise/luci/cr-buildbucket.cfg",
	"fuchsia,infra/config/generated/fuchsia/luci/cr-buildbucket.cfg",
	}
	}

	if err := c.commonFlags.Parse(); err != nil {
	return err
	}

	return nil
	}

	func (c *rptCmd) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
	fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
	return 1
	}

	if err := c.main(); err != nil {
	fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
	return 1
	}
	return 0
	}

	func (c *rptCmd) main() error {
	if !c.verbose {
	log.SetOutput(io.Discard)
	}

	ctx := context.Background()

	buildClient, err := buildbucket.NewBuildsClient(
	ctx,
	buildbucketHost,
	c.commonFlags.parsedAuthOpts,
	)
	if err != nil {
	return err
	}

	authClient, err := auth.NewAuthenticator(ctx, auth.OptionalLogin, c.commonFlags.parsedAuthOpts).Client()
	if err != nil {
	return fmt.Errorf("failed to get authenticated http client: %w", err)
	}

	gitilesClient, err := gitiles.NewClient(c.LUCIConfigHost, c.LUCIConfigProject, authClient)
	if err != nil {
	return err
	}

	lucicfg, err := c.getLUCIConfigs(ctx, *gitilesClient)
	if err != nil {
	return err
	}

	gerritClient, err := gerrit.NewClient(c.gerritHost, c.gerritProject, authClient)
	if err != nil {
	return err
	}

	if err = c.getChangeDetails(ctx, *gerritClient); err != nil {
	return err
	}

	availablePresubmitBuilders := c.getAvailablePresubmitBuilders(lucicfg, c.gerritHost, c.gerritProject)

	triggeredBuilds, err := c.getTriggeredBuildsForChange(ctx, buildClient)
	if err != nil {
	return err
	}

	triggeredBuilds = c.filterFailedBuilders(triggeredBuilds)
	missingBuilders := c.getMissingBuilders(availablePresubmitBuilders, triggeredBuilds)
	return c.triggerMissingBuilders(ctx, buildClient, missingBuilders)
	}

	// getLUCIConfigs fetches the LUCI configuration files contained at
	// the project specified in c.LUCIConfigProject and c.LUCIConfigPath
	func (c *rptCmd) getLUCIConfigs(ctx context.Context, gitilesClient gitiles.Client) (LUCIConfigurationFiles, error) {
	log.Println("Downloading LUCI configuration files...")
	// Set non-nil values for the configs.
	commitQueueConfig := &cvpb.Config{}
	crBuildBucketConfigs := map[string]*buildbucketpb.BuildbucketCfg{}

	commitQueueFileContents, err := gitilesClient.DownloadFile(
	ctx,
	c.CommitQueueCfgPath,
	"refs/heads/main",
	)
	if err != nil {
	return LUCIConfigurationFiles{}, err
	}

	if err := proto.UnmarshalText(commitQueueFileContents, commitQueueConfig); err != nil {
	return LUCIConfigurationFiles{}, err
	}

	for _, projectPath := range c.CrBuildBucketCfgPaths {
	projectPathSplit := strings.Split(projectPath, ",")
	if len(projectPathSplit) != 2 {
	return LUCIConfigurationFiles{}, errors.New(
	fmt.Sprintf(
	"Invalid entry to cr-buildbucket-cfg-project-path, expected 'project,path/to/cr-buildbucket.cfg' got %s",
	projectPath,
	),
	)
	}
	project := projectPathSplit[0]
	path := projectPathSplit[1]

	cfg := &buildbucketpb.BuildbucketCfg{}
	CrBuildBucketFileContents, err := gitilesClient.DownloadFile(
	ctx,
	path,
	"refs/heads/main",
	)
	if err != nil {
	return LUCIConfigurationFiles{}, err
	}

	if err = proto.UnmarshalText(CrBuildBucketFileContents, cfg); err != nil {
	return LUCIConfigurationFiles{}, err
	}

	crBuildBucketConfigs[project] = cfg
	}

	return LUCIConfigurationFiles{
	commitQueueConfig: commitQueueConfig,
	crBuildBucketConfigs: crBuildBucketConfigs,
	}, nil
	}

	// getChangeDetails queries Gerrit for the target change to determine
	// the ChangeRef, ChangeRepo, Project and Patchset (if not specified)
	func (c *rptCmd) getChangeDetails(ctx context.Context, gerritClient gerrit.Client) error {
	change, err := gerritClient.GetChange(
	ctx,
	c.gerritChangeID,
	gerritpb.QueryOption_ALL_REVISIONS,
	)
	if err != nil {
	return err
	}

	c.gerritPatchset = int64(change.Revisions[change.CurrentRevision].Number)
	c.gerritProject = change.Project

	return nil
	}

	// getAvailablePresubmitBuilders parses through available builders in
	// presubmit and if an equivalent builder is required in postsubmit, adds it to
	// the list of builders we check to trigger.
	func (c rptCmd) getAvailablePresubmitBuilders(lucicfg LUCIConfigurationFiles, gerritHost string, gerritProject string) []buildbucketpb.BuilderID {
	// Throw buckets into a map for cross referencing.
	flaggedBuildersByBucketAndProjectMap := map[string]map[string]map[string]bool{}
	for project, cfg := range lucicfg.crBuildBucketConfigs {
	if _, ok := flaggedBuildersByBucketAndProjectMap[project]; !ok {
	flaggedBuildersByBucketAndProjectMap[project] = map[string]map[string]bool{}
	}
	for _, bucket := range cfg.GetBuckets() {
	flaggedBuildersByBucketAndProjectMap[project][bucket.GetName()] = getBuildersFlaggedForRpt(bucket)
	}
	}

	// Find CQ for the target change.
	configGroups := lucicfg.commitQueueConfig.GetConfigGroups()
	var availablePresubmitBuilders []*buildbucketpb.BuilderID

	for _, cfg := range configGroups {
	projectMatch := cfg.GetGerrit()[0].GetProjects()[0].GetName() == gerritProject
	cfgUrl, _ := url.Parse(cfg.GetGerrit()[0].GetUrl())
	hostMatch := cfgUrl.Host == gerritHost
	if projectMatch && hostMatch {
	for _, builder := range cfg.GetVerifiers().GetTryjob().GetBuilders() {
	builderSplit := strings.Split(builder.GetName(), "/")
	builderID := &buildbucketpb.BuilderID{
	Project: builderSplit[0],
	Bucket: builderSplit[1],
	Builder: builderSplit[2],
	}

	if c.allPresubmit \|\| flaggedBuildersByBucketAndProjectMap[builderID.Project][builderID.Bucket][builderID.Builder] {
	availablePresubmitBuilders = append(availablePresubmitBuilders, builderID)
	}
	}
	}
	}

	// Sort for logging output clarity.
	sort.SliceStable(availablePresubmitBuilders, func(i, j int) bool {
	return availablePresubmitBuilders[i].Builder < availablePresubmitBuilders[j].Builder
	})

	return availablePresubmitBuilders
	}

	// getTriggeredBuildersForChange calls the buildbucket SearchBuilds RPC
	// and reports back all builders that have already triggered against the change.
	func (c rptCmd) getTriggeredBuildsForChange(ctx context.Context, buildClient buildbucketpb.BuildsClient) ([]buildbucketpb.Build, error) {
	// Pull all builds triggered against the CL.
	resp, err := buildClient.SearchBuilds(
	ctx,
	&buildbucketpb.SearchBuildsRequest{
	Predicate: &buildbucketpb.BuildPredicate{
	GerritChanges: []*buildbucketpb.GerritChange{
	{
	Host: c.gerritHost,
	Project: c.gerritProject,
	Change: c.gerritChangeID,
	Patchset: c.gerritPatchset,
	},
	},
	},
	Fields: &field_mask.FieldMask{Paths: []string{
	"builds.*.builder",
	"builds.*.status",
	}},
	PageSize: 1000,
	},
	)
	if err != nil {
	return []*buildbucketpb.Build{}, err
	}

	return resp.GetBuilds(), nil
	}

	// filterFailedBuilders filters failed builders out of the triggered builders
	// so that they will be re-triggered by run-postsubmit-tryjobs.
	func (c rptCmd) filterFailedBuilders(triggeredBuilds []buildbucketpb.Build) []*buildbucketpb.Build {
	var filteredBuilds []*buildbucketpb.Build
	for _, build := range triggeredBuilds {
	if !slices.Contains(
	[]buildbucketpb.Status{
	buildbucketpb.Status_FAILURE,
	buildbucketpb.Status_INFRA_FAILURE,
	},
	build.GetStatus(),
	) {
	filteredBuilds = append(filteredBuilds, build)
	}
	}
	return filteredBuilds
	}

	// getMissingBuilders finds the diff between availablePresubmitBuilders and the
	// builds already triggered by the change.
	func (c rptCmd) getMissingBuilders(availablePresubmitBuilders []buildbucketpb.BuilderID, triggeredPresubmitBuilds []buildbucketpb.Build) []buildbucketpb.BuilderID {
	var missingBuilders []*buildbucketpb.BuilderID
	var triggeredBuilders []string
	for _, build := range triggeredPresubmitBuilds {
	triggeredBuilders = append(triggeredBuilders, build.GetBuilder().GetBuilder())
	}

	// Find which builders haven't been triggered yet.
	for _, builder := range availablePresubmitBuilders {
	if !slices.Contains(triggeredBuilders, builder.GetBuilder()) && !builderSliceContains(missingBuilders, builder) {
	missingBuilders = append(missingBuilders, builder)
	}
	}

	return missingBuilders
	}

	// triggerMissingBuilders triggers the diff between all builders available
	// and the builders that have already been triggered for the change.
	func (c rptCmd) triggerMissingBuilders(ctx context.Context, buildClient buildbucketpb.BuildsClient, missingBuilders []buildbucketpb.BuilderID) error {
	// If we have no builders to trigger, return.
	if len(missingBuilders) == 0 {
	log.Println("No builders found to trigger.")
	return nil
	}

	// Print out list of builders to be triggered for logging.
	for _, builder := range missingBuilders {
	log.Printf("Builder: %s\n", builder.GetBuilder())
	}

	// Require command line confirmation. Automated tools should pass the -f flag.
	if !(c.force \|\| c.dryRun) {
	// The cost has been calculated roughly on average as of 2022 to be $0.57 per builder triggered
	// GCE Cost: $0.53/h
	// Orchestrator builder: 0.45h
	// Subbuild builder: 0.45h
	// Testing Subtasks: 0.17h
	fmt.Printf("You are about to trigger %d tryjobs with an estimated cost of $%.2f USD. Additionally, this will likely consume inelastic resources. Do you wish to proceed? (y/n)\n", len(missingBuilders), 0.57*float64(len(missingBuilders)))
	confirm := ""
	fmt.Scanln(&confirm)
	if !slices.Contains([]string{"y", "Y", "yes", "Yes", "YES"}, confirm) {
	fmt.Println("Process aborted.")
	return nil
	} else {
	fmt.Println("You can skip this confirmation next time by passing the -f flag.")
	}
	}

	// Assemble requests into a batch.
	var requestBatch []*buildbucketpb.BatchRequest_Request
	var triggeredBuilders []string
	for _, builder := range missingBuilders {
	req := &buildbucketpb.BatchRequest_Request{
	Request: &buildbucketpb.BatchRequest_Request_ScheduleBuild{
	ScheduleBuild: &buildbucketpb.ScheduleBuildRequest{
	Builder: builder,
	GerritChanges: []*buildbucketpb.GerritChange{
	{
	Host: c.gerritHost,
	Project: c.gerritProject,
	Change: c.gerritChangeID,
	Patchset: c.gerritPatchset,
	},
	},
	// Slightly below default priority.
	Priority: 31,
	},
	},
	}
	triggeredBuilders = append(triggeredBuilders, builder.Builder)
	requestBatch = append(requestBatch, req)

	}

	if !c.dryRun {
	resp, err := buildClient.Batch(
	ctx,
	&buildbucketpb.BatchRequest{
	Requests: requestBatch,
	},
	)
	if err != nil {
	return err
	}

	jsonResp, err := json.MarshalIndent(resp, "", " ")
	if err != nil {
	return err
	}

	log.Printf(
	"BatchRequest response: \n%s",
	jsonResp,
	)
	}

	if c.jsonOutputFile != "" {
	// Assemble builder data into easy to consume json output.
	jsonOutput := &BuildersToTrigger{
	Builders: triggeredBuilders,
	}

	// Marshal output to json.
	var rawJSON []byte
	var err error
	var f io.WriteCloser = os.Stdout
	if c.jsonOutputFile != "-" {
	if f, err = os.Create(c.jsonOutputFile); err != nil {
	return err
	}
	defer f.Close()
	}

	if c.jsonOutputFile == "-" {
	rawJSON, err = json.MarshalIndent(jsonOutput, "", " ")
	rawJSON = append(rawJSON, '\n')
	} else {
	rawJSON, err = json.Marshal(jsonOutput)
	rawJSON = append(rawJSON, '\n')
	}

	if err != nil {
	return err
	}

	if _, err := f.Write(rawJSON); err != nil {
	return err
	}
	}

	return nil
	}

	// slices.Contains doesn't really work with slices of pointers, so this helper
	// performs a low effort check for inclusion.
	func builderSliceContains(slice []buildbucketpb.BuilderID, builder buildbucketpb.BuilderID) bool {
	for _, b := range slice {
	if b.GetBuilder() == builder.GetBuilder() {
	return true
	}
	}
	return false
	}

	// getBuildersFlaggedForRpt finds builders with the `run_postsubmit_tryjobs_include` property.
	func getBuildersFlaggedForRpt(bucket *buildbucketpb.Bucket) map[string]bool {
	flaggedBuilders := map[string]bool{}
	for _, b := range bucket.GetSwarming().GetBuilders() {
	properties := &structpb.Struct{}
	protojson.Unmarshal([]byte(b.GetProperties()), properties)
	if builderTags, ok := properties.GetFields()["$fuchsia/builder_tags"]; ok {
	if runPostsubmitTryjobsInclude, ok := builderTags.GetStructValue().GetFields()["run_postsubmit_tryjobs_include"]; ok {
	flaggedBuilders[b.GetName()] = runPostsubmitTryjobsInclude.GetBoolValue()
	} else {
	flaggedBuilders[b.GetName()] = false
	}
	} else {
	flaggedBuilders[b.GetName()] = false
	}
	}

	return flaggedBuilders
	}