cmd/recipe_bootstrap/main.go - infra/infra - Git at Google

 // Copyright 2019 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 package main

 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"

 	"github.com/golang/protobuf/proto"
 	"go.chromium.org/luci/common/retry"
 	"go.chromium.org/luci/common/retry/transient"
 	"go.fuchsia.dev/infra/checkout"
 	rbc "go.fuchsia.dev/infra/cmd/recipe_bootstrap/checkout"
 	"go.fuchsia.dev/infra/cmd/recipe_bootstrap/manifest"
 	"go.fuchsia.dev/infra/cmd/recipe_bootstrap/props"
 	"go.fuchsia.dev/infra/gerrit"
 	"go.fuchsia.dev/infra/gitiles"
 	"google.golang.org/protobuf/encoding/prototext"
 	"google.golang.org/protobuf/types/known/structpb"

 	"go.chromium.org/luci/auth"
 	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
 	gerritapi "go.chromium.org/luci/common/api/gerrit"
 	gitilesapi "go.chromium.org/luci/common/api/gitiles"
 	"go.chromium.org/luci/common/logging"
 	"go.chromium.org/luci/common/logging/gologger"
 	gerritpb "go.chromium.org/luci/common/proto/gerrit"
 	"go.chromium.org/luci/logdog/client/butlerlib/bootstrap"
 	"go.chromium.org/luci/logdog/client/butlerlib/streamclient"
 	"go.chromium.org/luci/luciexe"
 )

 const (
 	// Fallback ref to fetch, when build has no input, e.g. when the build is manually
 	// triggered with LUCI scheduler, or when the recipes_host_override property is set.
 	fallbackRef = "refs/heads/master"

 	// Default integration.git remote, for when build has no input, e.g. when
 	// the build is manually triggered with LUCI scheduler.
 	defaultIntegrationRemote = "https://fuchsia.googlesource.com/integration"

 	// Recipe manifest filename relative to root of an integration checkout.
 	recipesManifest = "infra/recipes"

 	// Recipes.git remote.
 	recipesRemote = "https://fuchsia.googlesource.com/infra/recipes"

 	// Default timeout for checkout operations.
 	defaultCheckoutTimeout = 5 * time.Minute
 )

 // resolveRef returns a resolved sha1 from a ref e.g. refs/heads/main.
 func resolveRef(ctx context.Context, host, project, ref string) (string, error) {
 	authClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{Scopes: []string{gitilesapi.OAuthScope}}).Client()
 	if err != nil {
 		return "", fmt.Errorf("could not initialize auth client: %w", err)
 	}
 	client, err := gitiles.NewClient(ctx, host, project, authClient)
 	if err != nil {
 		return "", fmt.Errorf("could not initialize gitiles client: %w", err)
 	}
 	return client.LatestCommit(ctx, ref)
 }

 // resolveGitilesRef returns the ref associated with the build's input.
 //
 // If the build input has a GitilesCommit, return GitilesCommit.Ref.
 //
 // If the build input only has a GerritChange, query change info from Gerrit and
 // return the ref.
 // Note that this is the ref that the change targets i.e. refs/heads/*, not the
 // magic change ref, e.g. refs/changes/*.
 //
 // If the build input has neither, return the fallbackRef.
 func resolveGitilesRef(ctx context.Context, buildInput *buildbucketpb.Build_Input, fallbackRef string) (string, error) {
 	if buildInput.GitilesCommit != nil {
 		return buildInput.GitilesCommit.Ref, nil
 	}
 	if len(buildInput.GerritChanges) > 0 {
 		authClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{Scopes: []string{gerritapi.OAuthScope}}).Client()
 		if err != nil {
 			return "", fmt.Errorf("could not initialize auth client: %w", err)
 		}
 		client, err := gerrit.NewClient(ctx, buildInput.GerritChanges[0].Host, buildInput.GerritChanges[0].Project, authClient)
 		if err != nil {
 			return "", fmt.Errorf("failed to initialize gerrit client: %w", err)
 		}
 		// GetChange may sometimes encounter transient failures which should be
 		// retried.
 		retryPolicy := transient.Only(func() retry.Iterator {
 			return &retry.ExponentialBackoff{
 				Limited: retry.Limited{
 					Delay:   5 * time.Second,
 					Retries: 3,
 				},
 				Multiplier: 1.5,
 			}
 		})
 		var resp *gerritpb.ChangeInfo
 		if err := retry.Retry(ctx, retryPolicy, func() error {
 			resp, err = client.GetChange(ctx, buildInput.GerritChanges[0].Change)
 			return err
 		}, nil); err != nil {
 			return "", fmt.Errorf("failed to get change info: %w", err)
 		}
 		return resp.Ref, nil
 	}
 	return fallbackRef, nil
 }

 // ensureGitilesCommit ensures that the incoming build always has an
 // Input.GitilesCommit. This ensures that this build and any child builds
 // always use a consistent HEAD.
 //
 // The host and project for the GitilesCommit are determined by the
 // `ResolveRepo` strategy, with the fallback remote being either the default
 // integration remote set in this file, or the override set by the
 // `recipe_integration_remote` property.
 func ensureGitilesCommit(ctx context.Context, build *buildbucketpb.Build) error {
 	if build.Input.GitilesCommit != nil {
 		return nil
 	}
 	fallbackRemote, err := props.GetStringInputProperty(build, "recipe_integration_remote")
 	if err != nil {
 		return err
 	}
 	if fallbackRemote == "" {
 		fallbackRemote = defaultIntegrationRemote
 	}
 	host, project, err := rbc.ResolveRepo(build.Input, fallbackRemote)
 	if err != nil {
 		return fmt.Errorf("failed to resolve host and project: %w", err)
 	}
 	ref, err := resolveGitilesRef(ctx, build.Input, fallbackRef)
 	if err != nil {
 		return fmt.Errorf("failed to resolve gitiles ref: %w", err)
 	}
 	revision, err := resolveRef(ctx, host, project, ref)
 	if err != nil {
 		return fmt.Errorf("failed to resolve %s HEAD: %v", ref, err)
 	}
 	build.Input.GitilesCommit = &buildbucketpb.GitilesCommit{
 		Host:    host,
 		Project: project,
 		Id:      revision,
 		Ref:     ref,
 	}
 	return nil
 }

 // getBuilderFilePath returns the builder file path to read the build input properties.
 func getBuilderFilePath(integrationDir string, build *buildbucketpb.Build) string {
 	// TODO(fxbug.dev//71232): switch to "builders" directory once all maintained release branches are newer.
 	return filepath.Join(
 		integrationDir, "infra", "config", "generated", build.Builder.Project,
 		"for_review_only", "buildbucket", build.Builder.Bucket,
 		fmt.Sprintf("%s.textproto", build.Builder.Builder),
 	)
 }

 // checkoutIntegration checks out the integration.git repo and returns the path
 // to the directory containing the checkout.
 func checkoutIntegration(ctx context.Context, build *buildbucketpb.Build) (string, error) {
 	// Do a checkout in a temporary directory to avoid checkout conflicts with
 	// the recipe's working directory.
 	cwd, err := os.Getwd()
 	if err != nil {
 		return "", err
 	}
 	integrationDir, err := ioutil.TempDir(cwd, "integration-checkout")
 	if err != nil {
 		return integrationDir, err
 	}
 	integrationURL, integrationBaseRevision, err := resolveIntegration(ctx, build)
 	if err != nil {
 		return integrationDir, err
 	}
 	if err := props.SetInputProperty(build, "integration_base_revision", integrationBaseRevision); err != nil {
 		return integrationDir, err
 	}
 	tctx, cancel := context.WithTimeout(ctx, defaultCheckoutTimeout)
 	defer cancel()
 	return integrationDir, checkout.Checkout(tctx, build.Input, *integrationURL, integrationBaseRevision, integrationDir)
 }

 // resolveIntegration resolves the integration URL and revision to checkout.
 //
 // If the `recipes_host_override` property is specified, then override the
 // integration host.
 //
 // If the `integration_base_revision` property is set, use this revision.
 func resolveIntegration(ctx context.Context, build *buildbucketpb.Build) (*url.URL, string, error) {
 	recipesHostOverride, err := props.GetStringInputProperty(build, "recipes_host_override")
 	if err != nil {
 		return nil, "", err
 	}
 	integrationURL, err := rbc.ResolveIntegrationURL(build.Input, recipesHostOverride)
 	if err != nil {
 		return nil, "", err
 	}
 	revision, err := props.GetStringInputProperty(build, "integration_base_revision")
 	if err != nil {
 		return nil, "", err
 	}
 	if revision == "" {
 		commit := build.Input.GitilesCommit
 		if integrationURL.Host == commit.Host && integrationURL.Path == commit.Project {
 			revision = build.Input.GitilesCommit.Id
 		} else {
 			var ref string
 			// This assumes that for every ref `refs/heads/foo` of every repository
 			// pinned in integration that we care about submitting changes to, there
 			// exists a corresponding ref `refs/heads/foo` in the integration repository.
 			if recipesHostOverride == "" {
 				ref = build.Input.GitilesCommit.Ref
 			} else {
 				ref = fallbackRef
 			}
 			revision, err = resolveRef(ctx, integrationURL.Host, integrationURL.Path, ref)
 			if err != nil {
 				return nil, "", fmt.Errorf("failed to resolve integration HEAD: %w", err)
 			}
 		}

 	}
 	return integrationURL, revision, nil
 }

 // resolveBuildProperties resolves the build input properties to use.
 //
 // If enable_property_versioning is set, merge the build input properties with
 // integration properties.
 func resolveBuildProperties(builderFilePath string, build *buildbucketpb.Build) error {
 	enablePropertyVersion, err := props.GetBoolInputProperty(build, "enable_property_versioning")
 	if err != nil {
 		return err
 	}
 	// If the enable_property_versioning is not set, we'll use the original input properties.
 	if !enablePropertyVersion {
 		return nil
 	}
 	// Make a copy of the original input properties.
 	originalProperties := build.Input.Properties.AsMap()
 	builderBytes, err := ioutil.ReadFile(builderFilePath)
 	if err != nil {
 		return fmt.Errorf("failed to read builder file %s: %v", builderFilePath, err)
 	}
 	// Replace the input properties with the resolved integration.git properties.
 	build.Input.Properties, err = builderProperties(builderBytes)
 	if err != nil {
 		return fmt.Errorf("failed to read builder properties from integration repo: %w", err)
 	}
 	// Merge the original input properties, which should contain only request properties, into integration.git properties.
 	for property, value := range originalProperties {
 		if err := props.SetInputProperty(build, property, value); err != nil {
 			return fmt.Errorf("failed to set the property %s: %v", property, err)
 		}
 	}
 	return nil
 }

 // Gets builder properties from buildbucketpb.Bucket bytes data.
 func builderProperties(manifestBytes []byte) (*structpb.Struct, error) {
 	builderProto := &buildbucketpb.Bucket{}
 	if err := prototext.Unmarshal(manifestBytes, builderProto); err != nil {
 		return nil, err
 	}
 	builders := builderProto.GetSwarming().GetBuilders()
 	if len(builders) != 1 {
 		return nil, fmt.Errorf("expected 1 builder, got %d", len(builders))
 	}
 	properties := builders[0].GetProperties()
 	data := &structpb.Struct{}
 	if err := json.Unmarshal([]byte(properties), data); err != nil {
 		return nil, err
 	}
 	return data, nil
 }

 // checkoutRecipes resolves the recipe version to check out, checks out the recipes
 // repo, and returns the path to the checked-out repository.
 //
 // If the build input has a recipe change, then checkout directly using the
 // build input.
 //
 // Otherwise, checkout at the revision specified by Build.Input.Properties.
 func checkoutRecipes(ctx context.Context, integrationDir string, build *buildbucketpb.Build) (string, error) {
 	// Do recipes checkout in a temporary directory to avoid checkout conflicts
 	// with the recipe's working directory.
 	cwd, err := os.Getwd()
 	if err != nil {
 		return "", fmt.Errorf("failed to get working directory: %w", err)

 	}
 	recipesDir, err := ioutil.TempDir(cwd, "recipes-checkout")
 	if err != nil {
 		return recipesDir, fmt.Errorf("failed to get create tempdir: %w", err)

 	}
 	recipesURL, err := url.Parse(recipesRemote)
 	if err != nil {
 		return recipesDir, fmt.Errorf("could not parse URL %s", recipesRemote)
 	}
 	var input *buildbucketpb.Build_Input
 	hasRecipeChange, err := rbc.HasRepoChange(build.Input, recipesRemote)
 	if err != nil {
 		return recipesDir, fmt.Errorf("could not determine whether build input has recipe change: %w", err)
 	}
 	if hasRecipeChange {
 		input = build.Input
 	} else {
 		// Use the recipe version from the checkout integrationDir.
 		manifestXML, err := os.Open(filepath.Join(integrationDir, recipesManifest))
 		if err != nil {
 			return recipesDir, fmt.Errorf("failed to read recipes manifest file: %s: %v", filepath.Join(integrationDir, recipesManifest), err)
 		}
 		defer manifestXML.Close()
 		proj, err := manifest.ResolveRecipesProject(manifestXML, recipesRemote)
 		if err != nil {
 			return recipesDir, err
 		}
 		input = &buildbucketpb.Build_Input{
 			GitilesCommit: &buildbucketpb.GitilesCommit{
 				Host:    recipesURL.Host,
 				Project: strings.TrimLeft(recipesURL.Path, "/"),
 				Id:      proj.Revision,
 			},
 		}
 	}
 	tctx, cancel := context.WithTimeout(ctx, defaultCheckoutTimeout)
 	defer cancel()
 	if err := checkout.Checkout(tctx, input, *recipesURL, "", recipesDir); err != nil {
 		return recipesDir, fmt.Errorf("failed to checkout: %w", err)
 	}
 	return recipesDir, nil
 }

 // fetchRecipeDeps runs `recipes.py fetch`, which checks out all recipe git
 // repos that the current recipes repo repends on. `recipes.py luciexe` would
 // handle fetching deps even if we didn't run `recipes.py fetch` separately
 // beforehand, but running `recipes.py fetch` separately makes it easier to
 // distinguish between fetch errors (which are generally caused by transient Git
 // server issues) and true build errors within the executed recipe.
 func fetchRecipeDeps(ctx context.Context, recipesDir string) error {
 	cmd, err := recipeCommand(ctx, recipesDir, "fetch")
 	if err != nil {
 		return err
 	}
 	if err := cmd.Run(); err != nil {
 		return fmt.Errorf("failed to fetch recipe deps: %w", err)
 	}
 	return nil
 }

 // executeRecipe exec's `recipes.py luciexe`, passing build.proto through stdin.
 func executeRecipe(ctx context.Context, recipesDir string, build *buildbucketpb.Build, outputPath string) error {
 	args := []string{"luciexe"}
 	if outputPath != "" {
 		args = append(args, "--output", outputPath)
 	}
 	cmd, err := recipeCommand(ctx, recipesDir, args...)
 	if err != nil {
 		return err
 	}
 	inputData, err := proto.Marshal(build)
 	if err != nil {
 		return fmt.Errorf("could not marshal input build: %w", err)
 	}
 	cmd.Stdin = bytes.NewBuffer(inputData)
 	return cmd.Run()
 }

 // recipeCommand constructs an exec.Cmd that will run the `recipes.py` script
 // found in `recipesDir` with the specified arguments.
 func recipeCommand(ctx context.Context, recipesDir string, recipesPyArgs ...string) (*exec.Cmd, error) {
 	args := []string{filepath.Join(recipesDir, "recipes.py")}
 	args = append(args, recipesPyArgs...)
 	python, err := exec.LookPath("python")
 	if err != nil {
 		return nil, fmt.Errorf("could not find python on PATH: %w", err)
 	}
 	cmd := exec.CommandContext(ctx, python, args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	return cmd, nil
 }

 // outputBuildSummary outputs error details to the current build's
 // summary_markdown in case of failure.
 func outputBuildSummary(ctx context.Context, summary string) error {
 	bootstrap, err := bootstrap.Get()
 	if err != nil {
 		return err
 	}
 	stream, err := bootstrap.Client.NewDatagramStream(
 		ctx,
 		luciexe.BuildProtoStreamSuffix,
 		streamclient.WithContentType(luciexe.BuildProtoContentType),
 	)
 	if err != nil {
 		return err
 	}
 	build := &buildbucketpb.Build{}
 	build.SummaryMarkdown = summary
 	build.Status = buildbucketpb.Status_INFRA_FAILURE
 	outputData, err := proto.Marshal(build)
 	if err != nil {
 		return fmt.Errorf("failed to marshal output build.proto: %w", err)
 	}
 	return stream.WriteDatagram(outputData)
 }

 // setUpRecipes processes the build input to determine which revision of recipes
 // to checkout, then checks out the recipes repo at that revision. It returns
 // the deserialized input build.proto along with the path to the recipes
 // checkout.
 func setUpRecipes(ctx context.Context) (build *buildbucketpb.Build, recipesDir string, err error) {
 	data, err := ioutil.ReadAll(os.Stdin)
 	if err != nil {
 		err = fmt.Errorf("failed to read build.proto from stdin: %w", err)
 		return
 	}
 	build = &buildbucketpb.Build{}
 	if err = proto.Unmarshal(data, build); err != nil {
 		err = fmt.Errorf("failed to unmarshal build.proto from stdin: %w", err)
 		return
 	}
 	if err = ensureGitilesCommit(ctx, build); err != nil {
 		err = fmt.Errorf("failed to ensure that build has a gitiles commit: %w", err)
 		return
 	}
 	integrationDir, err := checkoutIntegration(ctx, build)
 	defer os.RemoveAll(integrationDir)
 	if err != nil {
 		err = fmt.Errorf("failed to checkout integration bundle: %w", err)
 		return
 	}
 	if err = resolveBuildProperties(getBuilderFilePath(integrationDir, build), build); err != nil {
 		err = fmt.Errorf("failed to resolve the build properties: %w", err)
 		return
 	}
 	recipesDir, err = checkoutRecipes(ctx, integrationDir, build)
 	if err != nil {
 		err = fmt.Errorf("failed to download recipe bundle: %w", err)
 		return
 	}
 	if err = fetchRecipeDeps(ctx, recipesDir); err != nil {
 		return
 	}
 	b, err := json.MarshalIndent(build.Input.Properties, "", "  ")
 	if err != nil {
 		err = fmt.Errorf("failed to marshal build input properties: %w", err)
 		return
 	}
 	// TODO(fxbug.dev/72956): Emit the build properties to a separate output log once
 	// it's possible for recipe_bootstrap to emit build.proto to the same Logdog stream
 	// as the recipe engine.
 	logging.Infof(ctx, "resolved input properties: %s", string(b))
 	return build, recipesDir, nil
 }

 func execute(ctx context.Context) error {
 	outputPath := flag.String("output", "", "Path to write the final build.proto state to.")
 	flag.Parse()

 	build, recipesDir, err := setUpRecipes(ctx)
 	// `recipesDir` might not exist depending on the setup failure mode, so ignore any
 	// error when removing it.
 	defer os.RemoveAll(recipesDir)
 	if err != nil {
 		if outputErr := outputBuildSummary(ctx, err.Error()); outputErr != nil {
 			logging.Errorf(ctx, "Failed to output build summary after recipe setup failed: %v", outputErr)
 		}
 		return err
 	}
 	if err := executeRecipe(ctx, recipesDir, build, *outputPath); err != nil {
 		return fmt.Errorf("failed to execute recipe: %w", err)
 	}
 	return nil
 }

 func main() {
 	ctx := context.Background()
 	ctx = gologger.StdConfig.Use(ctx)
 	if err := execute(ctx); err != nil {
 		logging.Errorf(ctx, err.Error())
 		os.Exit(1)
 	}
 }
	// Copyright 2019 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	package main

	import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.fuchsia.dev/infra/checkout"
	rbc "go.fuchsia.dev/infra/cmd/recipe_bootstrap/checkout"
	"go.fuchsia.dev/infra/cmd/recipe_bootstrap/manifest"
	"go.fuchsia.dev/infra/cmd/recipe_bootstrap/props"
	"go.fuchsia.dev/infra/gerrit"
	"go.fuchsia.dev/infra/gitiles"
	"google.golang.org/protobuf/encoding/prototext"
	"google.golang.org/protobuf/types/known/structpb"

	"go.chromium.org/luci/auth"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	gerritapi "go.chromium.org/luci/common/api/gerrit"
	gitilesapi "go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	gerritpb "go.chromium.org/luci/common/proto/gerrit"
	"go.chromium.org/luci/logdog/client/butlerlib/bootstrap"
	"go.chromium.org/luci/logdog/client/butlerlib/streamclient"
	"go.chromium.org/luci/luciexe"
	)

	const (
	// Fallback ref to fetch, when build has no input, e.g. when the build is manually
	// triggered with LUCI scheduler, or when the recipes_host_override property is set.
	fallbackRef = "refs/heads/master"

	// Default integration.git remote, for when build has no input, e.g. when
	// the build is manually triggered with LUCI scheduler.
	defaultIntegrationRemote = "https://fuchsia.googlesource.com/integration"

	// Recipe manifest filename relative to root of an integration checkout.
	recipesManifest = "infra/recipes"

	// Recipes.git remote.
	recipesRemote = "https://fuchsia.googlesource.com/infra/recipes"

	// Default timeout for checkout operations.
	defaultCheckoutTimeout = 5 * time.Minute
	)

	// resolveRef returns a resolved sha1 from a ref e.g. refs/heads/main.
	func resolveRef(ctx context.Context, host, project, ref string) (string, error) {
	authClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{Scopes: []string{gitilesapi.OAuthScope}}).Client()
	if err != nil {
	return "", fmt.Errorf("could not initialize auth client: %w", err)
	}
	client, err := gitiles.NewClient(ctx, host, project, authClient)
	if err != nil {
	return "", fmt.Errorf("could not initialize gitiles client: %w", err)
	}
	return client.LatestCommit(ctx, ref)
	}

	// resolveGitilesRef returns the ref associated with the build's input.
	//
	// If the build input has a GitilesCommit, return GitilesCommit.Ref.
	//
	// If the build input only has a GerritChange, query change info from Gerrit and
	// return the ref.
	// Note that this is the ref that the change targets i.e. refs/heads/*, not the
	// magic change ref, e.g. refs/changes/*.
	//
	// If the build input has neither, return the fallbackRef.
	func resolveGitilesRef(ctx context.Context, buildInput *buildbucketpb.Build_Input, fallbackRef string) (string, error) {
	if buildInput.GitilesCommit != nil {
	return buildInput.GitilesCommit.Ref, nil
	}
	if len(buildInput.GerritChanges) > 0 {
	authClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{Scopes: []string{gerritapi.OAuthScope}}).Client()
	if err != nil {
	return "", fmt.Errorf("could not initialize auth client: %w", err)
	}
	client, err := gerrit.NewClient(ctx, buildInput.GerritChanges[0].Host, buildInput.GerritChanges[0].Project, authClient)
	if err != nil {
	return "", fmt.Errorf("failed to initialize gerrit client: %w", err)
	}
	// GetChange may sometimes encounter transient failures which should be
	// retried.
	retryPolicy := transient.Only(func() retry.Iterator {
	return &retry.ExponentialBackoff{
	Limited: retry.Limited{
	Delay: 5 * time.Second,
	Retries: 3,
	},
	Multiplier: 1.5,
	}
	})
	var resp *gerritpb.ChangeInfo
	if err := retry.Retry(ctx, retryPolicy, func() error {
	resp, err = client.GetChange(ctx, buildInput.GerritChanges[0].Change)
	return err
	}, nil); err != nil {
	return "", fmt.Errorf("failed to get change info: %w", err)
	}
	return resp.Ref, nil
	}
	return fallbackRef, nil
	}

	// ensureGitilesCommit ensures that the incoming build always has an
	// Input.GitilesCommit. This ensures that this build and any child builds
	// always use a consistent HEAD.
	//
	// The host and project for the GitilesCommit are determined by the
	// `ResolveRepo` strategy, with the fallback remote being either the default
	// integration remote set in this file, or the override set by the
	// `recipe_integration_remote` property.
	func ensureGitilesCommit(ctx context.Context, build *buildbucketpb.Build) error {
	if build.Input.GitilesCommit != nil {
	return nil
	}
	fallbackRemote, err := props.GetStringInputProperty(build, "recipe_integration_remote")
	if err != nil {
	return err
	}
	if fallbackRemote == "" {
	fallbackRemote = defaultIntegrationRemote
	}
	host, project, err := rbc.ResolveRepo(build.Input, fallbackRemote)
	if err != nil {
	return fmt.Errorf("failed to resolve host and project: %w", err)
	}
	ref, err := resolveGitilesRef(ctx, build.Input, fallbackRef)
	if err != nil {
	return fmt.Errorf("failed to resolve gitiles ref: %w", err)
	}
	revision, err := resolveRef(ctx, host, project, ref)
	if err != nil {
	return fmt.Errorf("failed to resolve %s HEAD: %v", ref, err)
	}
	build.Input.GitilesCommit = &buildbucketpb.GitilesCommit{
	Host: host,
	Project: project,
	Id: revision,
	Ref: ref,
	}
	return nil
	}

	// getBuilderFilePath returns the builder file path to read the build input properties.
	func getBuilderFilePath(integrationDir string, build *buildbucketpb.Build) string {
	// TODO(fxbug.dev//71232): switch to "builders" directory once all maintained release branches are newer.
	return filepath.Join(
	integrationDir, "infra", "config", "generated", build.Builder.Project,
	"for_review_only", "buildbucket", build.Builder.Bucket,
	fmt.Sprintf("%s.textproto", build.Builder.Builder),
	)
	}

	// checkoutIntegration checks out the integration.git repo and returns the path
	// to the directory containing the checkout.
	func checkoutIntegration(ctx context.Context, build *buildbucketpb.Build) (string, error) {
	// Do a checkout in a temporary directory to avoid checkout conflicts with
	// the recipe's working directory.
	cwd, err := os.Getwd()
	if err != nil {
	return "", err
	}
	integrationDir, err := ioutil.TempDir(cwd, "integration-checkout")
	if err != nil {
	return integrationDir, err
	}
	integrationURL, integrationBaseRevision, err := resolveIntegration(ctx, build)
	if err != nil {
	return integrationDir, err
	}
	if err := props.SetInputProperty(build, "integration_base_revision", integrationBaseRevision); err != nil {
	return integrationDir, err
	}
	tctx, cancel := context.WithTimeout(ctx, defaultCheckoutTimeout)
	defer cancel()
	return integrationDir, checkout.Checkout(tctx, build.Input, *integrationURL, integrationBaseRevision, integrationDir)
	}

	// resolveIntegration resolves the integration URL and revision to checkout.
	//
	// If the `recipes_host_override` property is specified, then override the
	// integration host.
	//
	// If the `integration_base_revision` property is set, use this revision.
	func resolveIntegration(ctx context.Context, build buildbucketpb.Build) (url.URL, string, error) {
	recipesHostOverride, err := props.GetStringInputProperty(build, "recipes_host_override")
	if err != nil {
	return nil, "", err
	}
	integrationURL, err := rbc.ResolveIntegrationURL(build.Input, recipesHostOverride)
	if err != nil {
	return nil, "", err
	}
	revision, err := props.GetStringInputProperty(build, "integration_base_revision")
	if err != nil {
	return nil, "", err
	}
	if revision == "" {
	commit := build.Input.GitilesCommit
	if integrationURL.Host == commit.Host && integrationURL.Path == commit.Project {
	revision = build.Input.GitilesCommit.Id
	} else {
	var ref string
	// This assumes that for every ref `refs/heads/foo` of every repository
	// pinned in integration that we care about submitting changes to, there
	// exists a corresponding ref `refs/heads/foo` in the integration repository.
	if recipesHostOverride == "" {
	ref = build.Input.GitilesCommit.Ref
	} else {
	ref = fallbackRef
	}
	revision, err = resolveRef(ctx, integrationURL.Host, integrationURL.Path, ref)
	if err != nil {
	return nil, "", fmt.Errorf("failed to resolve integration HEAD: %w", err)
	}
	}

	}
	return integrationURL, revision, nil
	}

	// resolveBuildProperties resolves the build input properties to use.
	//
	// If enable_property_versioning is set, merge the build input properties with
	// integration properties.
	func resolveBuildProperties(builderFilePath string, build *buildbucketpb.Build) error {
	enablePropertyVersion, err := props.GetBoolInputProperty(build, "enable_property_versioning")
	if err != nil {
	return err
	}
	// If the enable_property_versioning is not set, we'll use the original input properties.
	if !enablePropertyVersion {
	return nil
	}
	// Make a copy of the original input properties.
	originalProperties := build.Input.Properties.AsMap()
	builderBytes, err := ioutil.ReadFile(builderFilePath)
	if err != nil {
	return fmt.Errorf("failed to read builder file %s: %v", builderFilePath, err)
	}
	// Replace the input properties with the resolved integration.git properties.
	build.Input.Properties, err = builderProperties(builderBytes)
	if err != nil {
	return fmt.Errorf("failed to read builder properties from integration repo: %w", err)
	}
	// Merge the original input properties, which should contain only request properties, into integration.git properties.
	for property, value := range originalProperties {
	if err := props.SetInputProperty(build, property, value); err != nil {
	return fmt.Errorf("failed to set the property %s: %v", property, err)
	}
	}
	return nil
	}

	// Gets builder properties from buildbucketpb.Bucket bytes data.
	func builderProperties(manifestBytes []byte) (*structpb.Struct, error) {
	builderProto := &buildbucketpb.Bucket{}
	if err := prototext.Unmarshal(manifestBytes, builderProto); err != nil {
	return nil, err
	}
	builders := builderProto.GetSwarming().GetBuilders()
	if len(builders) != 1 {
	return nil, fmt.Errorf("expected 1 builder, got %d", len(builders))
	}
	properties := builders[0].GetProperties()
	data := &structpb.Struct{}
	if err := json.Unmarshal([]byte(properties), data); err != nil {
	return nil, err
	}
	return data, nil
	}

	// checkoutRecipes resolves the recipe version to check out, checks out the recipes
	// repo, and returns the path to the checked-out repository.
	//
	// If the build input has a recipe change, then checkout directly using the
	// build input.
	//
	// Otherwise, checkout at the revision specified by Build.Input.Properties.
	func checkoutRecipes(ctx context.Context, integrationDir string, build *buildbucketpb.Build) (string, error) {
	// Do recipes checkout in a temporary directory to avoid checkout conflicts
	// with the recipe's working directory.
	cwd, err := os.Getwd()
	if err != nil {
	return "", fmt.Errorf("failed to get working directory: %w", err)

	}
	recipesDir, err := ioutil.TempDir(cwd, "recipes-checkout")
	if err != nil {
	return recipesDir, fmt.Errorf("failed to get create tempdir: %w", err)

	}
	recipesURL, err := url.Parse(recipesRemote)
	if err != nil {
	return recipesDir, fmt.Errorf("could not parse URL %s", recipesRemote)
	}
	var input *buildbucketpb.Build_Input
	hasRecipeChange, err := rbc.HasRepoChange(build.Input, recipesRemote)
	if err != nil {
	return recipesDir, fmt.Errorf("could not determine whether build input has recipe change: %w", err)
	}
	if hasRecipeChange {
	input = build.Input
	} else {
	// Use the recipe version from the checkout integrationDir.
	manifestXML, err := os.Open(filepath.Join(integrationDir, recipesManifest))
	if err != nil {
	return recipesDir, fmt.Errorf("failed to read recipes manifest file: %s: %v", filepath.Join(integrationDir, recipesManifest), err)
	}
	defer manifestXML.Close()
	proj, err := manifest.ResolveRecipesProject(manifestXML, recipesRemote)
	if err != nil {
	return recipesDir, err
	}
	input = &buildbucketpb.Build_Input{
	GitilesCommit: &buildbucketpb.GitilesCommit{
	Host: recipesURL.Host,
	Project: strings.TrimLeft(recipesURL.Path, "/"),
	Id: proj.Revision,
	},
	}
	}
	tctx, cancel := context.WithTimeout(ctx, defaultCheckoutTimeout)
	defer cancel()
	if err := checkout.Checkout(tctx, input, *recipesURL, "", recipesDir); err != nil {
	return recipesDir, fmt.Errorf("failed to checkout: %w", err)
	}
	return recipesDir, nil
	}

	// fetchRecipeDeps runs `recipes.py fetch`, which checks out all recipe git
	// repos that the current recipes repo repends on. `recipes.py luciexe` would
	// handle fetching deps even if we didn't run `recipes.py fetch` separately
	// beforehand, but running `recipes.py fetch` separately makes it easier to
	// distinguish between fetch errors (which are generally caused by transient Git
	// server issues) and true build errors within the executed recipe.
	func fetchRecipeDeps(ctx context.Context, recipesDir string) error {
	cmd, err := recipeCommand(ctx, recipesDir, "fetch")
	if err != nil {
	return err
	}
	if err := cmd.Run(); err != nil {
	return fmt.Errorf("failed to fetch recipe deps: %w", err)
	}
	return nil
	}

	// executeRecipe exec's `recipes.py luciexe`, passing build.proto through stdin.
	func executeRecipe(ctx context.Context, recipesDir string, build *buildbucketpb.Build, outputPath string) error {
	args := []string{"luciexe"}
	if outputPath != "" {
	args = append(args, "--output", outputPath)
	}
	cmd, err := recipeCommand(ctx, recipesDir, args...)
	if err != nil {
	return err
	}
	inputData, err := proto.Marshal(build)
	if err != nil {
	return fmt.Errorf("could not marshal input build: %w", err)
	}
	cmd.Stdin = bytes.NewBuffer(inputData)
	return cmd.Run()
	}

	// recipeCommand constructs an exec.Cmd that will run the `recipes.py` script
	// found in `recipesDir` with the specified arguments.
	func recipeCommand(ctx context.Context, recipesDir string, recipesPyArgs ...string) (*exec.Cmd, error) {
	args := []string{filepath.Join(recipesDir, "recipes.py")}
	args = append(args, recipesPyArgs...)
	python, err := exec.LookPath("python")
	if err != nil {
	return nil, fmt.Errorf("could not find python on PATH: %w", err)
	}
	cmd := exec.CommandContext(ctx, python, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd, nil
	}

	// outputBuildSummary outputs error details to the current build's
	// summary_markdown in case of failure.
	func outputBuildSummary(ctx context.Context, summary string) error {
	bootstrap, err := bootstrap.Get()
	if err != nil {
	return err
	}
	stream, err := bootstrap.Client.NewDatagramStream(
	ctx,
	luciexe.BuildProtoStreamSuffix,
	streamclient.WithContentType(luciexe.BuildProtoContentType),
	)
	if err != nil {
	return err
	}
	build := &buildbucketpb.Build{}
	build.SummaryMarkdown = summary
	build.Status = buildbucketpb.Status_INFRA_FAILURE
	outputData, err := proto.Marshal(build)
	if err != nil {
	return fmt.Errorf("failed to marshal output build.proto: %w", err)
	}
	return stream.WriteDatagram(outputData)
	}

	// setUpRecipes processes the build input to determine which revision of recipes
	// to checkout, then checks out the recipes repo at that revision. It returns
	// the deserialized input build.proto along with the path to the recipes
	// checkout.
	func setUpRecipes(ctx context.Context) (build *buildbucketpb.Build, recipesDir string, err error) {
	data, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
	err = fmt.Errorf("failed to read build.proto from stdin: %w", err)
	return
	}
	build = &buildbucketpb.Build{}
	if err = proto.Unmarshal(data, build); err != nil {
	err = fmt.Errorf("failed to unmarshal build.proto from stdin: %w", err)
	return
	}
	if err = ensureGitilesCommit(ctx, build); err != nil {
	err = fmt.Errorf("failed to ensure that build has a gitiles commit: %w", err)
	return
	}
	integrationDir, err := checkoutIntegration(ctx, build)
	defer os.RemoveAll(integrationDir)
	if err != nil {
	err = fmt.Errorf("failed to checkout integration bundle: %w", err)
	return
	}
	if err = resolveBuildProperties(getBuilderFilePath(integrationDir, build), build); err != nil {
	err = fmt.Errorf("failed to resolve the build properties: %w", err)
	return
	}
	recipesDir, err = checkoutRecipes(ctx, integrationDir, build)
	if err != nil {
	err = fmt.Errorf("failed to download recipe bundle: %w", err)
	return
	}
	if err = fetchRecipeDeps(ctx, recipesDir); err != nil {
	return
	}
	b, err := json.MarshalIndent(build.Input.Properties, "", " ")
	if err != nil {
	err = fmt.Errorf("failed to marshal build input properties: %w", err)
	return
	}
	// TODO(fxbug.dev/72956): Emit the build properties to a separate output log once
	// it's possible for recipe_bootstrap to emit build.proto to the same Logdog stream
	// as the recipe engine.
	logging.Infof(ctx, "resolved input properties: %s", string(b))
	return build, recipesDir, nil
	}

	func execute(ctx context.Context) error {
	outputPath := flag.String("output", "", "Path to write the final build.proto state to.")
	flag.Parse()

	build, recipesDir, err := setUpRecipes(ctx)
	// `recipesDir` might not exist depending on the setup failure mode, so ignore any
	// error when removing it.
	defer os.RemoveAll(recipesDir)
	if err != nil {
	if outputErr := outputBuildSummary(ctx, err.Error()); outputErr != nil {
	logging.Errorf(ctx, "Failed to output build summary after recipe setup failed: %v", outputErr)
	}
	return err
	}
	if err := executeRecipe(ctx, recipesDir, build, *outputPath); err != nil {
	return fmt.Errorf("failed to execute recipe: %w", err)
	}
	return nil
	}

	func main() {
	ctx := context.Background()
	ctx = gologger.StdConfig.Use(ctx)
	if err := execute(ctx); err != nil {
	logging.Errorf(ctx, err.Error())
	os.Exit(1)
	}
	}