| // Copyright 2019 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| package main |
| |
| import ( |
| "bytes" |
| "context" |
| "encoding/json" |
| "errors" |
| "fmt" |
| "io/ioutil" |
| "net/url" |
| "os" |
| "os/exec" |
| "path/filepath" |
| |
| "go.chromium.org/luci/auth" |
| buildbucketpb "go.chromium.org/luci/buildbucket/proto" |
| gerritapi "go.chromium.org/luci/common/api/gerrit" |
| gitilesapi "go.chromium.org/luci/common/api/gitiles" |
| "go.chromium.org/luci/common/logging" |
| "go.chromium.org/luci/common/logging/gologger" |
| "go.chromium.org/luci/logdog/client/butlerlib/bootstrap" |
| "go.chromium.org/luci/logdog/client/butlerlib/streamclient" |
| "go.chromium.org/luci/luciexe" |
| "go.chromium.org/luci/luciexe/exe" |
| "google.golang.org/protobuf/encoding/protojson" |
| "google.golang.org/protobuf/encoding/prototext" |
| "google.golang.org/protobuf/proto" |
| "google.golang.org/protobuf/types/known/structpb" |
| |
| "go.fuchsia.dev/infra/checkout" |
| rbc "go.fuchsia.dev/infra/cmd/recipe_bootstrap/checkout" |
| "go.fuchsia.dev/infra/cmd/recipe_bootstrap/props" |
| "go.fuchsia.dev/infra/cmd/recipe_bootstrap/recipes" |
| "go.fuchsia.dev/infra/gerrit" |
| "go.fuchsia.dev/infra/gitiles" |
| ) |
| |
| const ( |
| // Fallback ref to fetch, when build has no input, e.g. when the build is manually |
| // triggered with LUCI scheduler, or when the recipes_host_override property is set. |
| fallbackRef = "refs/heads/main" |
| |
| // Default integration.git remote, for when build has no input, e.g. when |
| // the build is manually triggered with LUCI scheduler. |
| defaultIntegrationRemote = "https://fuchsia.googlesource.com/integration" |
| ) |
| |
| // resolveRef returns a resolved sha1 from a ref e.g. refs/heads/main. |
| func resolveRef(ctx context.Context, host, project, ref string) (string, error) { |
| authClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{Scopes: []string{gitilesapi.OAuthScope}}).Client() |
| if err != nil { |
| return "", fmt.Errorf("could not initialize auth client: %w", err) |
| } |
| client, err := gitiles.NewClient(host, project, authClient) |
| if err != nil { |
| return "", fmt.Errorf("could not initialize gitiles client: %w", err) |
| } |
| return client.LatestCommit(ctx, ref) |
| } |
| |
| // resolveGitilesRef returns the ref associated with the build's input. |
| // |
| // If the build input has a GitilesCommit, return GitilesCommit.Ref. |
| // |
| // If the build input only has a GerritChange, query change info from Gerrit and |
| // return the ref. |
| // Note that this is the ref that the change targets i.e. refs/heads/*, not the |
| // magic change ref, e.g. refs/changes/*. |
| // |
| // If the build input has neither, return the fallbackRef. |
| func resolveGitilesRef(ctx context.Context, buildInput *buildbucketpb.Build_Input, fallbackRef string) (string, error) { |
| if buildInput.GitilesCommit != nil { |
| return buildInput.GitilesCommit.Ref, nil |
| } |
| if len(buildInput.GerritChanges) > 0 { |
| authClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{Scopes: []string{gerritapi.OAuthScope}}).Client() |
| if err != nil { |
| return "", fmt.Errorf("could not initialize auth client: %w", err) |
| } |
| client, err := gerrit.NewClient(buildInput.GerritChanges[0].Host, buildInput.GerritChanges[0].Project, authClient) |
| if err != nil { |
| return "", fmt.Errorf("failed to initialize gerrit client: %w", err) |
| } |
| resp, err := client.GetChange(ctx, buildInput.GerritChanges[0].Change) |
| if err != nil { |
| return "", err |
| } |
| return resp.Ref, nil |
| } |
| return fallbackRef, nil |
| } |
| |
| // ensureGitilesCommit ensures that the incoming build always has an |
| // Input.GitilesCommit. This ensures that this build and any child builds |
| // always use a consistent HEAD. |
| // |
| // The host and project for the GitilesCommit are determined by the |
| // `ResolveRepo` strategy, with the fallback remote being either the default |
| // integration remote set in this file, or the override set by the |
| // `recipe_integration_remote` property. |
| func ensureGitilesCommit(ctx context.Context, build *buildbucketpb.Build) error { |
| if build.Input.GitilesCommit != nil { |
| return nil |
| } |
| fallbackRemote, err := props.GetStringInputProperty(build, "recipe_integration_remote") |
| if err != nil { |
| return err |
| } |
| if fallbackRemote == "" { |
| fallbackRemote = defaultIntegrationRemote |
| } |
| host, project, err := rbc.ResolveRepo(build.Input, fallbackRemote) |
| if err != nil { |
| return fmt.Errorf("failed to resolve host and project: %w", err) |
| } |
| ref, err := resolveGitilesRef(ctx, build.Input, fallbackRef) |
| if err != nil { |
| return fmt.Errorf("failed to resolve gitiles ref: %w", err) |
| } |
| revision, err := resolveRef(ctx, host, project, ref) |
| if err != nil { |
| return fmt.Errorf("failed to resolve %s HEAD: %w", ref, err) |
| } |
| build.Input.GitilesCommit = &buildbucketpb.GitilesCommit{ |
| Host: host, |
| Project: project, |
| Id: revision, |
| Ref: ref, |
| } |
| return nil |
| } |
| |
| // checkoutIntegration checks out the integration.git repo and returns the path |
| // to the directory containing the checkout. |
| func checkoutIntegration(ctx context.Context, build *buildbucketpb.Build) (string, error) { |
| // Do a checkout in a temporary directory to avoid checkout conflicts with |
| // the recipe's working directory. |
| cwd, err := os.Getwd() |
| if err != nil { |
| return "", err |
| } |
| integrationDir, err := ioutil.TempDir(cwd, "integration-checkout") |
| if err != nil { |
| return integrationDir, err |
| } |
| integrationURL, integrationBaseRevision, err := resolveIntegration(ctx, build) |
| if err != nil { |
| return integrationDir, err |
| } |
| if err := props.SetInputProperty(build, "integration_base_revision", integrationBaseRevision); err != nil { |
| return integrationDir, err |
| } |
| tctx, cancel := context.WithTimeout(ctx, rbc.DefaultCheckoutTimeout) |
| defer cancel() |
| if err := checkout.Checkout(tctx, build.Input, *integrationURL, integrationBaseRevision, integrationDir); err != nil { |
| return integrationDir, fmt.Errorf("failed to checkout integration repo: %w", err) |
| } |
| return integrationDir, nil |
| } |
| |
| // resolveIntegration resolves the integration URL and revision to checkout. |
| // |
| // If the `recipes_host_override` property is specified, then override the |
| // integration host. |
| // |
| // If the `integration_base_revision` property is set, use this revision. |
| func resolveIntegration(ctx context.Context, build *buildbucketpb.Build) (*url.URL, string, error) { |
| recipesHostOverride, err := props.GetStringInputProperty(build, "recipes_host_override") |
| if err != nil { |
| return nil, "", err |
| } |
| integrationURL, err := rbc.ResolveIntegrationURL(build.Input, recipesHostOverride) |
| if err != nil { |
| return nil, "", err |
| } |
| |
| baseRevisionProperty, err := props.GetStringInputProperty(build, "integration_base_revision") |
| if err != nil { |
| return nil, "", err |
| } |
| if baseRevisionProperty != "" { |
| // If `integration_base_revision` is set then we're probably running as |
| // a subbuild and `integration_base_revision` is the revision resolved |
| // by the parent build, so we should use the same revision. |
| return integrationURL, baseRevisionProperty, nil |
| } |
| |
| commit := build.Input.GitilesCommit |
| if integrationURL.Host == commit.Host && integrationURL.Path == commit.Project { |
| // Either the build was triggered on an integration change and we |
| // already resolved a base revision with `ensureGitilesCommit`, or the |
| // build was triggered by a specific integration commit. In either case |
| // we want to use that resolved revision to ensure we check out the same |
| // version of integration as the recipe. |
| return integrationURL, commit.Id, nil |
| } |
| |
| // Otherwise we haven't yet resolved an integration base revision, so we |
| // need to choose the correct integration ref to resolve and then resolve |
| // that ref to a revision. |
| |
| overrideRef, err := props.GetStringInputProperty(build, "recipes_integration_ref_override") |
| if err != nil { |
| return nil, "", err |
| } |
| var ref string |
| if overrideRef != "" { |
| ref = overrideRef |
| } else if commit.Ref != "" { |
| // This assumes that for every ref `refs/heads/foo` of every repository |
| // pinned in integration that we care about submitting changes to, there |
| // exists a corresponding ref `refs/heads/foo` in the integration |
| // repository. |
| ref = commit.Ref |
| } else { |
| ref = fallbackRef |
| } |
| |
| revision, err := resolveRef(ctx, integrationURL.Host, integrationURL.Path, ref) |
| if err != nil { |
| return nil, "", fmt.Errorf("failed to resolve integration HEAD: %w", err) |
| } |
| |
| return integrationURL, revision, nil |
| } |
| |
| // resolveBuildProperties resolves the build input properties to use by reading |
| // the builder's properties from integration.git and merging them with the input |
| // properties. |
| func resolveBuildProperties(ctx context.Context, integrationDir string, build *buildbucketpb.Build) error { |
| versionedProperties, err := readVersionedProperties(ctx, integrationDir, build.Builder) |
| if err != nil { |
| return fmt.Errorf("failed to read builder properties from integration repo: %w", err) |
| } |
| // Make a copy of the original input properties. |
| originalProperties := build.Input.Properties.AsMap() |
| // Replace the input properties with the resolved versioned properties. |
| build.Input.Properties = versionedProperties |
| // Merge the original input properties (primarily request properties) into |
| // the versioned properties, letting the request properties take precedence. |
| return exe.WriteProperties(build.Input.Properties, originalProperties) |
| } |
| |
| // readVersionedProperties loads the builder's properties from an integration |
| // checkout. |
| // |
| // Depending on how recent this integration.git checkout is, the properties may |
| // come from one of several possible files. Older versions of integration.git |
| // stored properties at one of two paths in textproto files that conformed to |
| // the `Bucket` schema, whereas modern versions of integration.git store |
| // properties in dedicated JSON files. |
| // |
| // We prefer the most modern format available. |
| func readVersionedProperties(ctx context.Context, integrationDir string, builder *buildbucketpb.BuilderID) (*structpb.Struct, error) { |
| // Possible files from which we might be able to read properties. We want to |
| // prioritize more modern files, since older files might be malformed in |
| // later revisions that also have modern files. Hence we use an ordered |
| // slice of structs rather than an unordered map. |
| // TODO(olivernewman): Remove support for reading properties from bucket |
| // files once we no longer care about building versions of integration.git |
| // that don't contain JSON property files. |
| possiblePropertyFiles := []struct { |
| path string |
| // loadFunc specifies how to load properties from such a file into a |
| // proto struct. |
| loadFunc func([]byte, *structpb.Struct) error |
| }{ |
| { |
| path: filepath.Join("properties", builder.Bucket, builder.Builder+".json"), |
| loadFunc: loadPropertiesFromJSONFile, |
| }, |
| { |
| path: filepath.Join("builders", builder.Bucket, builder.Builder+".textproto"), |
| loadFunc: loadPropertiesFromBucketFile, |
| }, |
| { |
| path: filepath.Join("for_review_only", "buildbucket", builder.Bucket, builder.Builder+".textproto"), |
| loadFunc: loadPropertiesFromBucketFile, |
| }, |
| } |
| for _, f := range possiblePropertyFiles { |
| absPath := filepath.Join(generatedDirForBuilder(integrationDir, builder), f.path) |
| contents, err := ioutil.ReadFile(absPath) |
| if err != nil { |
| if os.IsNotExist(err) { |
| // If one file doesn't exist, fall back to a less modern format. |
| continue |
| } |
| return nil, err |
| } |
| properties := &structpb.Struct{} |
| if err := f.loadFunc(contents, properties); err != nil { |
| return nil, err |
| } |
| logging.Infof(ctx, "Using versioned properties from %s", absPath) |
| return properties, nil |
| } |
| return nil, fmt.Errorf("failed to locate properties file for builder %s", builder) |
| } |
| |
| // generatedDirForBuilder returns the path in an integration.git checkout to the |
| // directory containing all the config files for a builder's LUCI project. |
| func generatedDirForBuilder(integrationDir string, builder *buildbucketpb.BuilderID) string { |
| return filepath.Join(integrationDir, "infra", "config", "generated", builder.Project) |
| } |
| |
| // loadPropertiesFromBucketFile deserializes a builder's properties from a |
| // textproto file conforming to the `Bucket` schema. |
| func loadPropertiesFromBucketFile(contents []byte, properties *structpb.Struct) error { |
| builderProto := &buildbucketpb.Bucket{} |
| if err := prototext.Unmarshal(contents, builderProto); err != nil { |
| return err |
| } |
| builders := builderProto.GetSwarming().GetBuilders() |
| if len(builders) != 1 { |
| return fmt.Errorf("expected 1 builder, got %d", len(builders)) |
| } |
| propertiesJSON := builders[0].GetProperties() |
| return protojson.Unmarshal([]byte(propertiesJSON), properties) |
| } |
| |
| // loadPropertiesFromJSONFile deserializes a builder's properties from a JSON |
| // file containing a single JSON object corresponding to the properties. |
| func loadPropertiesFromJSONFile(contents []byte, properties *structpb.Struct) error { |
| return protojson.Unmarshal(contents, properties) |
| } |
| |
| // outputBuildSummary outputs error details to the current build's |
| // summary_markdown in case of failure. |
| func outputBuildSummary(ctx context.Context, buildErr error) error { |
| bootstrap, err := bootstrap.Get() |
| if err != nil { |
| return err |
| } |
| stream, err := bootstrap.Client.NewDatagramStream( |
| ctx, |
| luciexe.BuildProtoStreamSuffix, |
| streamclient.WithContentType(luciexe.BuildProtoContentType), |
| ) |
| if err != nil { |
| return err |
| } |
| defer stream.Close() |
| build := &buildbucketpb.Build{} |
| build.SummaryMarkdown = buildErr.Error() |
| // Check if the returned error declares whether or not it's an infra |
| // failure. If it doesn't declare one way or the other, we'll assume that it |
| // is an infra failure. |
| var maybeInfraErr interface{ IsInfraFailure() bool } |
| if errors.As(buildErr, &maybeInfraErr) && !maybeInfraErr.IsInfraFailure() { |
| build.Status = buildbucketpb.Status_FAILURE |
| } else { |
| build.Status = buildbucketpb.Status_INFRA_FAILURE |
| } |
| |
| outputData, err := proto.Marshal(build) |
| if err != nil { |
| return fmt.Errorf("failed to marshal output build.proto: %w", err) |
| } |
| return stream.WriteDatagram(outputData) |
| } |
| |
| // setUpRecipes processes the build input to determine which version of recipes |
| // to use (possibly from CAS rather than from Git if run by a led job) and |
| // downloads that version of the recipes. It returns a `RecipesExe` that can be |
| // used to run the recipes (whether they were downloaded from CAS or from Git) |
| // as a luciexe, along with the deserialized build.proto. |
| func setUpRecipes(ctx context.Context, recipesDir string) (recipes.RecipesExe, *buildbucketpb.Build, error) { |
| data, err := ioutil.ReadAll(os.Stdin) |
| if err != nil { |
| return nil, nil, fmt.Errorf("failed to read build.proto from stdin: %w", err) |
| } |
| build := &buildbucketpb.Build{} |
| if err := proto.Unmarshal(data, build); err != nil { |
| return nil, nil, fmt.Errorf("failed to unmarshal build.proto from stdin: %w", err) |
| } |
| if err := ensureGitilesCommit(ctx, build); err != nil { |
| return nil, nil, fmt.Errorf("failed to ensure that build has a gitiles commit: %w", err) |
| } |
| integrationDir, err := checkoutIntegration(ctx, build) |
| defer os.RemoveAll(integrationDir) |
| if err != nil { |
| return nil, nil, err |
| } |
| if err := resolveBuildProperties(ctx, integrationDir, build); err != nil { |
| return nil, nil, fmt.Errorf("failed to resolve the build properties: %w", err) |
| } |
| b, err := json.MarshalIndent(build.Input.Properties, "", " ") |
| if err != nil { |
| return nil, nil, fmt.Errorf("failed to marshal build input properties: %w", err) |
| } |
| usePy3, err := props.GetBoolInputProperty(build, "recipes_use_py3") |
| if err != nil { |
| return nil, nil, err |
| } |
| if usePy3 { |
| // TODO(fxbug.dev/83858): Set this env var unconditionally |
| // once we no longer need to support running recipes from release |
| // branches where they didn't yet support Python 3. |
| if err := os.Setenv("RECIPES_USE_PY3", "true"); err != nil { |
| return nil, nil, err |
| } |
| logging.Infof(ctx, "Using Python 3") |
| } |
| // TODO(fxbug.dev/72956): Emit the build properties to a separate output log |
| // once it's possible for recipe_bootstrap to emit build.proto to the same |
| // Logdog stream as the recipe engine. |
| logging.Infof(ctx, "Resolved input properties: %s", string(b)) |
| exe, err := recipes.DownloadRecipes(ctx, build, integrationDir, recipesDir) |
| if err != nil { |
| return nil, nil, err |
| } |
| return exe, build, nil |
| } |
| |
| func execute(ctx context.Context) error { |
| cwd, err := os.Getwd() |
| if err != nil { |
| return err |
| } |
| recipesDir, err := ioutil.TempDir(cwd, "recipes") |
| if err != nil { |
| return err |
| } |
| defer os.RemoveAll(recipesDir) |
| |
| exe, build, err := setUpRecipes(ctx, recipesDir) |
| if err != nil { |
| if outputErr := outputBuildSummary(ctx, err); outputErr != nil { |
| logging.Errorf(ctx, "Failed to output build summary after recipe setup failed: %s", outputErr) |
| } |
| return err |
| } |
| |
| commandLine, err := exe.LuciexeCommand() |
| if err != nil { |
| return err |
| } |
| // Forward any flags such as --output to the recipe. |
| commandLine = append(commandLine, os.Args[1:]...) |
| |
| cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...) |
| cmd.Stdout = os.Stdout |
| cmd.Stderr = os.Stderr |
| inputData, err := proto.Marshal(build) |
| if err != nil { |
| return fmt.Errorf("could not marshal input build: %w", err) |
| } |
| cmd.Stdin = bytes.NewBuffer(inputData) |
| logging.Infof(ctx, "Running luciexe command: %s", cmd) |
| if err := cmd.Run(); err != nil { |
| return fmt.Errorf("failed to execute recipe: %w", err) |
| } |
| return nil |
| } |
| |
| func main() { |
| ctx := context.Background() |
| ctx = gologger.StdConfig.Use(ctx) |
| if err := execute(ctx); err != nil { |
| logging.Errorf(ctx, err.Error()) |
| os.Exit(1) |
| } |
| } |