cmd/recipe_wrapper/main.go - infra/infra - Git at Google

 // Copyright 2019 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 package main

 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"

 	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
 	cipdclient "go.chromium.org/luci/cipd/client/cipd"
 	"go.chromium.org/luci/common/logging"
 	"go.chromium.org/luci/common/logging/gologger"
 	"go.chromium.org/luci/logdog/client/butlerlib/bootstrap"
 	"go.chromium.org/luci/logdog/client/butlerlib/streamclient"
 	"go.chromium.org/luci/luciexe"
 	"google.golang.org/protobuf/proto"

 	"go.fuchsia.dev/infra/cmd/recipe_wrapper/bcid"
 	"go.fuchsia.dev/infra/cmd/recipe_wrapper/cipd"
 	"go.fuchsia.dev/infra/cmd/recipe_wrapper/env"
 	"go.fuchsia.dev/infra/cmd/recipe_wrapper/props"
 	"go.fuchsia.dev/infra/cmd/recipe_wrapper/recipes"
 )

 // outputBuildSummary outputs error details to the current build's
 // summary_markdown in case of failure.
 func outputBuildSummary(ctx context.Context, buildErr error) error {
 	bootstrap, err := bootstrap.Get()
 	if err != nil {
 		return err
 	}
 	stream, err := bootstrap.Client.NewDatagramStream(
 		ctx,
 		luciexe.BuildProtoStreamSuffix,
 		streamclient.WithContentType(luciexe.BuildProtoContentType),
 	)
 	if err != nil {
 		return err
 	}
 	defer stream.Close()
 	build := &buildbucketpb.Build{}
 	build.SummaryMarkdown = buildErr.Error()
 	// Check if the returned error declares whether or not it's an infra
 	// failure. If it doesn't declare one way or the other, we'll assume that it
 	// is an infra failure.
 	var maybeInfraErr interface{ IsInfraFailure() bool }
 	if errors.As(buildErr, &maybeInfraErr) && !maybeInfraErr.IsInfraFailure() {
 		build.Status = buildbucketpb.Status_FAILURE
 	} else {
 		build.Status = buildbucketpb.Status_INFRA_FAILURE
 	}

 	outputData, err := proto.Marshal(build)
 	if err != nil {
 		return fmt.Errorf("failed to marshal output build.proto: %w", err)
 	}
 	return stream.WriteDatagram(outputData)
 }

 // initEnvironment sets up the environment with recipes, Python 2 etc.
 // Returns particulars about the environment created as well as a cleanup function
 // that the caller should run when the environment is no longer needed.
 func initEnvironment(ctx context.Context) (*env.Build, func(), error) {
 	cwd, err := os.Getwd()
 	if err != nil {
 		return nil, nil, err
 	}
 	recipesDir, err := os.MkdirTemp(cwd, "recipes")
 	if err != nil {
 		return nil, nil, err
 	}

 	exe, build, err := recipes.SetUp(ctx, recipesDir)
 	if err != nil {
 		if outputErr := outputBuildSummary(ctx, err); outputErr != nil {
 			logging.Errorf(ctx, "Failed to output build summary after recipe setup failed: %s", outputErr)
 		}
 		return nil, nil, err
 	}

 	// TODO(fxbug.dev/89307): Remove support for Python 2 once branches prior to
 	// f11 are no longer supported.
 	// Ideally this input property would be read in main() only (same as flags),
 	// but at this point it's simpler not to worry about it and just wait to
 	// delete this whole block.
 	noPy2, err := props.Bool(build, "no_python2")
 	if err != nil {
 		return nil, nil, err
 	}

 	pkgsToInstall := []cipd.Package{}
 	rootBinDir, err := os.MkdirTemp("", "recipe_wrapper")
 	if err != nil {
 		return nil, nil, err
 	}
 	if bcid.CanAttestPlatform() {
 		pkgsToInstall = append(pkgsToInstall, cipd.AttestToolPkg)
 	}
 	if noPy2 {
 		logging.Infof(ctx, "no_python2=true, not installing python 2")
 	} else {
 		logging.Infof(ctx, "no_python2=false, installing python 2")
 		pkgsToInstall = append(pkgsToInstall, cipd.VPythonPkg, cipd.CPythonPkg)
 	}
 	binDirs, err := cipd.Install(ctx, rootBinDir, pkgsToInstall...)
 	if err != nil {
 		return nil, nil, err
 	}
 	path := strings.Join(append(binDirs, os.Getenv("PATH")), string(os.PathListSeparator))
 	os.Setenv("PATH", path)

 	deferredStepsDir, err := os.MkdirTemp("", "bcid")
 	if err != nil {
 		return nil, nil, fmt.Errorf("couldn't create a deferred steps temp dir: %v", err)
 	}
 	if err := props.SetBuildInputProperty(build, "$fuchsia/recipe_wrapper",
 		map[string]any{"deferred_steps_dir": deferredStepsDir}); err != nil {
 		return nil, nil, fmt.Errorf("couldn't set `deferred_steps_dir` property: %v", err)
 	}
 	shouldUpload, err := props.Bool(build, "upload_bcid_attestation")
 	if err != nil {
 		logging.Infof(ctx, "couldn't find upload_bcid_attestation property, assuming false")
 		shouldUpload = false
 	}

 	logging.Infof(ctx, "Initialized execution environment to:\n%+v", os.Environ())

 	return &env.Build{
 			Environ:                 os.Environ(),
 			RecipesExe:              exe,
 			Build:                   build,
 			DeferredStepsDir:        deferredStepsDir,
 			AttestationShouldUpload: shouldUpload,
 		}, func() {
 			// No need to check errors here, trashing temp files is best effort.
 			os.RemoveAll(recipesDir)
 			os.RemoveAll(rootBinDir)
 			os.RemoveAll(deferredStepsDir)
 		}, err
 }

 func runRecipe(ctx context.Context, buildEnv *env.Build) error {
 	commandLine, err := buildEnv.RecipesExe.LuciexeCommand()
 	if err != nil {
 		return err
 	}
 	// Forward any flags such as --output to the recipe.
 	commandLine = append(commandLine, os.Args[1:]...)

 	cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	cmd.Env = buildEnv.Environ
 	inputData, err := proto.Marshal(buildEnv.Build)
 	if err != nil {
 		return fmt.Errorf("could not marshal input build: %w", err)
 	}
 	cmd.Stdin = bytes.NewBuffer(inputData)
 	logging.Infof(ctx, "Running luciexe command: %s", cmd)
 	if err := cmd.Run(); err != nil {
 		return fmt.Errorf("failed to execute recipe: %w", err)
 	}
 	return nil
 }

 func joinErrs(errs []error) string {
 	var errStrs []string
 	for _, err := range errs {
 		errStrs = append(errStrs, err.Error())
 	}
 	return strings.Join(errStrs, ",")
 }

 func pkgsInDir(ctx context.Context, dir string) ([]*cipd.PkgFile, error) {
 	pkgs := []*cipd.PkgFile{}
 	files, err := os.ReadDir(dir)
 	if err != nil {
 		return nil, err
 	}

 	logging.Infof(ctx, "Found %v deferred step files in %v", len(files), dir)
 	for _, f := range files {
 		// Annoyingly f is not an *os.File but an *fs.DirEntry,
 		// and *fs.DirEntries do not contain the filepath.
 		// It has to be joined to the directory it was read from.
 		path := filepath.Join(dir, f.Name())
 		data, err := os.ReadFile(path)
 		if err != nil {
 			return nil, fmt.Errorf("couldn't open %q: %v", path, err)
 		}
 		logging.Infof(ctx, "Read %v: %s", path, data)

 		pkgFile := &cipd.PkgFile{}
 		if err := json.Unmarshal(data, pkgFile); err != nil {
 			return nil, err
 		}
 		logging.Infof(ctx, "Unmarshaled to %+v", pkgFile)
 		pkgs = append(pkgs, pkgFile)
 	}

 	return pkgs, nil
 }

 // uploadAttestations will upload attestations to SCILo, then attach them as metadata to the CIPD packages.
 func uploadAttestations(ctx context.Context, buildEnv *env.Build, stmtBundles []*cipd.StmtBundle) error {
 	if !buildEnv.AttestationShouldUpload {
 		logging.Infof(ctx, "Attestation skipped, builder does not have `upload_bcid_attestation` set")
 		return nil
 	}

 	if err := uploadToSCILo(ctx, stmtBundles); err != nil {
 		return fmt.Errorf("unable to upload attestations to SCILo: %v", err)
 	}
 	// Attestations aren't valid if they didn't go in to SCILo, so only attach them as metadata _after_
 	// SCILo has accepted them.
 	if err := attachAttestations(ctx, stmtBundles); err != nil {
 		return fmt.Errorf("unable to attach attestations: %v", err)
 	}

 	return nil
 }

 func uploadToSCILo(ctx context.Context, stmtBundles []*cipd.StmtBundle) error {
 	for _, sb := range stmtBundles {
 		logging.Infof(ctx, "Uploading attestation for %v | %v:\n%+v",
 			sb.Package.PackageName, sb.Package.InstanceID, sb.Bundle)

 		_, err := bcid.UploadToSCILo(ctx, sb.Statement, sb.Bundle)
 		if err != nil {
 			return err
 		}
 	}

 	return nil
 }

 func attachAttestations(ctx context.Context, stmtBundles []*cipd.StmtBundle) error {
 	for _, sb := range stmtBundles {
 		logging.Infof(ctx, "Attaching attestation for %v | %v:\n%+v",
 			sb.Package.PackageName, sb.Package.InstanceID, sb.Bundle)
 		jsb, err := json.Marshal(sb.Bundle)
 		if err != nil {
 			return fmt.Errorf("unable to marshal %+v: %v", sb.Bundle, err)
 		}
 		cipd.AddMetadata(ctx, sb.Package.AsPin(), []cipdclient.Metadata{
 			{
 				Key:         "provenance",
 				Value:       jsb,
 				ContentType: "text/plain",
 			},
 		})
 	}

 	return nil
 }

 func attest(ctx context.Context, buildEnv *env.Build) error {
 	if !bcid.CanAttestPlatform() {
 		logging.Infof(ctx, "Unsupported attestation platform: %v/%v", runtime.GOOS, runtime.GOARCH)
 		return nil
 	}
 	pkgs, err := pkgsInDir(ctx, buildEnv.DeferredStepsDir)
 	if err != nil {
 		return err
 	}

 	logging.Infof(ctx, "Inspecting %v packages for attestation...", len(pkgs))
 	stmtBundles, errs := cipd.AttestPkgs(pkgs, bcid.ProdKeyID, buildEnv)
 	if len(errs) != 0 {
 		return fmt.Errorf("AttestPkgs returned errors: %v", joinErrs(errs))
 	}
 	if err := uploadAttestations(ctx, buildEnv, stmtBundles); err != nil {
 		return fmt.Errorf("unable to upload attestations: %v", err)
 	}

 	return nil
 }

 func main() {
 	ctx := context.Background()
 	ctx = gologger.StdConfig.Use(ctx)

 	logging.Infof(ctx, "Initializing build environment step")
 	env, cleanup, err := initEnvironment(ctx)
 	if err != nil {
 		logging.Errorf(ctx, fmt.Errorf("environment initialization failed: %v", err).Error())
 		os.Exit(1)
 	}
 	defer cleanup()

 	logging.Infof(ctx, "Running recipe step")
 	if err := runRecipe(ctx, env); err != nil {
 		logging.Errorf(ctx, fmt.Errorf("build failed: %v", err).Error())
 		os.Exit(1)
 	}

 	logging.Infof(ctx, "Running attestation step")
 	if err := attest(ctx, env); err != nil {
 		logging.Errorf(ctx, fmt.Errorf("build was successful, but couldn't attest: %v", err).Error())
 		os.Exit(1)
 	}
 }
	// Copyright 2019 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	package main

	import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	cipdclient "go.chromium.org/luci/cipd/client/cipd"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	"go.chromium.org/luci/logdog/client/butlerlib/bootstrap"
	"go.chromium.org/luci/logdog/client/butlerlib/streamclient"
	"go.chromium.org/luci/luciexe"
	"google.golang.org/protobuf/proto"

	"go.fuchsia.dev/infra/cmd/recipe_wrapper/bcid"
	"go.fuchsia.dev/infra/cmd/recipe_wrapper/cipd"
	"go.fuchsia.dev/infra/cmd/recipe_wrapper/env"
	"go.fuchsia.dev/infra/cmd/recipe_wrapper/props"
	"go.fuchsia.dev/infra/cmd/recipe_wrapper/recipes"
	)

	// outputBuildSummary outputs error details to the current build's
	// summary_markdown in case of failure.
	func outputBuildSummary(ctx context.Context, buildErr error) error {
	bootstrap, err := bootstrap.Get()
	if err != nil {
	return err
	}
	stream, err := bootstrap.Client.NewDatagramStream(
	ctx,
	luciexe.BuildProtoStreamSuffix,
	streamclient.WithContentType(luciexe.BuildProtoContentType),
	)
	if err != nil {
	return err
	}
	defer stream.Close()
	build := &buildbucketpb.Build{}
	build.SummaryMarkdown = buildErr.Error()
	// Check if the returned error declares whether or not it's an infra
	// failure. If it doesn't declare one way or the other, we'll assume that it
	// is an infra failure.
	var maybeInfraErr interface{ IsInfraFailure() bool }
	if errors.As(buildErr, &maybeInfraErr) && !maybeInfraErr.IsInfraFailure() {
	build.Status = buildbucketpb.Status_FAILURE
	} else {
	build.Status = buildbucketpb.Status_INFRA_FAILURE
	}

	outputData, err := proto.Marshal(build)
	if err != nil {
	return fmt.Errorf("failed to marshal output build.proto: %w", err)
	}
	return stream.WriteDatagram(outputData)
	}

	// initEnvironment sets up the environment with recipes, Python 2 etc.
	// Returns particulars about the environment created as well as a cleanup function
	// that the caller should run when the environment is no longer needed.
	func initEnvironment(ctx context.Context) (*env.Build, func(), error) {
	cwd, err := os.Getwd()
	if err != nil {
	return nil, nil, err
	}
	recipesDir, err := os.MkdirTemp(cwd, "recipes")
	if err != nil {
	return nil, nil, err
	}

	exe, build, err := recipes.SetUp(ctx, recipesDir)
	if err != nil {
	if outputErr := outputBuildSummary(ctx, err); outputErr != nil {
	logging.Errorf(ctx, "Failed to output build summary after recipe setup failed: %s", outputErr)
	}
	return nil, nil, err
	}

	// TODO(fxbug.dev/89307): Remove support for Python 2 once branches prior to
	// f11 are no longer supported.
	// Ideally this input property would be read in main() only (same as flags),
	// but at this point it's simpler not to worry about it and just wait to
	// delete this whole block.
	noPy2, err := props.Bool(build, "no_python2")
	if err != nil {
	return nil, nil, err
	}

	pkgsToInstall := []cipd.Package{}
	rootBinDir, err := os.MkdirTemp("", "recipe_wrapper")
	if err != nil {
	return nil, nil, err
	}
	if bcid.CanAttestPlatform() {
	pkgsToInstall = append(pkgsToInstall, cipd.AttestToolPkg)
	}
	if noPy2 {
	logging.Infof(ctx, "no_python2=true, not installing python 2")
	} else {
	logging.Infof(ctx, "no_python2=false, installing python 2")
	pkgsToInstall = append(pkgsToInstall, cipd.VPythonPkg, cipd.CPythonPkg)
	}
	binDirs, err := cipd.Install(ctx, rootBinDir, pkgsToInstall...)
	if err != nil {
	return nil, nil, err
	}
	path := strings.Join(append(binDirs, os.Getenv("PATH")), string(os.PathListSeparator))
	os.Setenv("PATH", path)

	deferredStepsDir, err := os.MkdirTemp("", "bcid")
	if err != nil {
	return nil, nil, fmt.Errorf("couldn't create a deferred steps temp dir: %v", err)
	}
	if err := props.SetBuildInputProperty(build, "$fuchsia/recipe_wrapper",
	map[string]any{"deferred_steps_dir": deferredStepsDir}); err != nil {
	return nil, nil, fmt.Errorf("couldn't set `deferred_steps_dir` property: %v", err)
	}
	shouldUpload, err := props.Bool(build, "upload_bcid_attestation")
	if err != nil {
	logging.Infof(ctx, "couldn't find upload_bcid_attestation property, assuming false")
	shouldUpload = false
	}

	logging.Infof(ctx, "Initialized execution environment to:\n%+v", os.Environ())

	return &env.Build{
	Environ: os.Environ(),
	RecipesExe: exe,
	Build: build,
	DeferredStepsDir: deferredStepsDir,
	AttestationShouldUpload: shouldUpload,
	}, func() {
	// No need to check errors here, trashing temp files is best effort.
	os.RemoveAll(recipesDir)
	os.RemoveAll(rootBinDir)
	os.RemoveAll(deferredStepsDir)
	}, err
	}

	func runRecipe(ctx context.Context, buildEnv *env.Build) error {
	commandLine, err := buildEnv.RecipesExe.LuciexeCommand()
	if err != nil {
	return err
	}
	// Forward any flags such as --output to the recipe.
	commandLine = append(commandLine, os.Args[1:]...)

	cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Env = buildEnv.Environ
	inputData, err := proto.Marshal(buildEnv.Build)
	if err != nil {
	return fmt.Errorf("could not marshal input build: %w", err)
	}
	cmd.Stdin = bytes.NewBuffer(inputData)
	logging.Infof(ctx, "Running luciexe command: %s", cmd)
	if err := cmd.Run(); err != nil {
	return fmt.Errorf("failed to execute recipe: %w", err)
	}
	return nil
	}

	func joinErrs(errs []error) string {
	var errStrs []string
	for _, err := range errs {
	errStrs = append(errStrs, err.Error())
	}
	return strings.Join(errStrs, ",")
	}

	func pkgsInDir(ctx context.Context, dir string) ([]*cipd.PkgFile, error) {
	pkgs := []*cipd.PkgFile{}
	files, err := os.ReadDir(dir)
	if err != nil {
	return nil, err
	}

	logging.Infof(ctx, "Found %v deferred step files in %v", len(files), dir)
	for _, f := range files {
	// Annoyingly f is not an os.File but an fs.DirEntry,
	// and *fs.DirEntries do not contain the filepath.
	// It has to be joined to the directory it was read from.
	path := filepath.Join(dir, f.Name())
	data, err := os.ReadFile(path)
	if err != nil {
	return nil, fmt.Errorf("couldn't open %q: %v", path, err)
	}
	logging.Infof(ctx, "Read %v: %s", path, data)

	pkgFile := &cipd.PkgFile{}
	if err := json.Unmarshal(data, pkgFile); err != nil {
	return nil, err
	}
	logging.Infof(ctx, "Unmarshaled to %+v", pkgFile)
	pkgs = append(pkgs, pkgFile)
	}

	return pkgs, nil
	}

	// uploadAttestations will upload attestations to SCILo, then attach them as metadata to the CIPD packages.
	func uploadAttestations(ctx context.Context, buildEnv env.Build, stmtBundles []cipd.StmtBundle) error {
	if !buildEnv.AttestationShouldUpload {
	logging.Infof(ctx, "Attestation skipped, builder does not have `upload_bcid_attestation` set")
	return nil
	}

	if err := uploadToSCILo(ctx, stmtBundles); err != nil {
	return fmt.Errorf("unable to upload attestations to SCILo: %v", err)
	}
	// Attestations aren't valid if they didn't go in to SCILo, so only attach them as metadata _after_
	// SCILo has accepted them.
	if err := attachAttestations(ctx, stmtBundles); err != nil {
	return fmt.Errorf("unable to attach attestations: %v", err)
	}

	return nil
	}

	func uploadToSCILo(ctx context.Context, stmtBundles []*cipd.StmtBundle) error {
	for _, sb := range stmtBundles {
	logging.Infof(ctx, "Uploading attestation for %v \| %v:\n%+v",
	sb.Package.PackageName, sb.Package.InstanceID, sb.Bundle)

	_, err := bcid.UploadToSCILo(ctx, sb.Statement, sb.Bundle)
	if err != nil {
	return err
	}
	}

	return nil
	}

	func attachAttestations(ctx context.Context, stmtBundles []*cipd.StmtBundle) error {
	for _, sb := range stmtBundles {
	logging.Infof(ctx, "Attaching attestation for %v \| %v:\n%+v",
	sb.Package.PackageName, sb.Package.InstanceID, sb.Bundle)
	jsb, err := json.Marshal(sb.Bundle)
	if err != nil {
	return fmt.Errorf("unable to marshal %+v: %v", sb.Bundle, err)
	}
	cipd.AddMetadata(ctx, sb.Package.AsPin(), []cipdclient.Metadata{
	{
	Key: "provenance",
	Value: jsb,
	ContentType: "text/plain",
	},
	})
	}

	return nil
	}

	func attest(ctx context.Context, buildEnv *env.Build) error {
	if !bcid.CanAttestPlatform() {
	logging.Infof(ctx, "Unsupported attestation platform: %v/%v", runtime.GOOS, runtime.GOARCH)
	return nil
	}
	pkgs, err := pkgsInDir(ctx, buildEnv.DeferredStepsDir)
	if err != nil {
	return err
	}

	logging.Infof(ctx, "Inspecting %v packages for attestation...", len(pkgs))
	stmtBundles, errs := cipd.AttestPkgs(pkgs, bcid.ProdKeyID, buildEnv)
	if len(errs) != 0 {
	return fmt.Errorf("AttestPkgs returned errors: %v", joinErrs(errs))
	}
	if err := uploadAttestations(ctx, buildEnv, stmtBundles); err != nil {
	return fmt.Errorf("unable to upload attestations: %v", err)
	}

	return nil
	}

	func main() {
	ctx := context.Background()
	ctx = gologger.StdConfig.Use(ctx)

	logging.Infof(ctx, "Initializing build environment step")
	env, cleanup, err := initEnvironment(ctx)
	if err != nil {
	logging.Errorf(ctx, fmt.Errorf("environment initialization failed: %v", err).Error())
	os.Exit(1)
	}
	defer cleanup()

	logging.Infof(ctx, "Running recipe step")
	if err := runRecipe(ctx, env); err != nil {
	logging.Errorf(ctx, fmt.Errorf("build failed: %v", err).Error())
	os.Exit(1)
	}

	logging.Infof(ctx, "Running attestation step")
	if err := attest(ctx, env); err != nil {
	logging.Errorf(ctx, fmt.Errorf("build was successful, but couldn't attest: %v", err).Error())
	os.Exit(1)
	}
	}