cmd/recipe_bootstrap/main.go

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"
	"go.fuchsia.dev/infra/cmd/build_init/checkout"
	rbc "go.fuchsia.dev/infra/cmd/recipe_bootstrap/checkout"
	"go.fuchsia.dev/infra/cmd/recipe_bootstrap/manifest"
	"go.fuchsia.dev/infra/cmd/recipe_bootstrap/props"
	"go.fuchsia.dev/infra/gitiles"

	"go.chromium.org/luci/auth"
	gitilesapi "go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"

	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
)

const (
	// Default integration ref to fetch, if not passed through build properties.
	defaultIntegrationRef = "HEAD"

	// Default integration.git remote, for when build has no input, e.g. when
	// the build is manually triggered with LUCI scheduler.
	defaultIntegrationRemote = "https://fuchsia.googlesource.com/integration"

	// Recipe manifest filename relative to root of an integration checkout.
	recipesManifest = "infra/recipes"

	// Recipes.git remote.
	recipesRemote = "https://fuchsia.googlesource.com/infra/recipes"

	// Default timeout for checkout operations.
	defaultCheckoutTimeout = 5 * time.Minute
)

// resolveHead returns a resolved sha1 from the HEAD of a repository.
func resolveHead(ctx context.Context, host, project string) (string, error) {
	authClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, auth.Options{Scopes: []string{gitilesapi.OAuthScope}}).Client()
	if err != nil {
		return "", fmt.Errorf("could not initialize auth client: %v", err)
	}
	client, err := gitiles.NewClient(ctx, host, project, authClient)
	if err != nil {
		return "", fmt.Errorf("could not initialize gitiles client: %v", err)
	}
	return client.LatestCommit(ctx)
}

// ensureGitilesCommit ensures that the incoming build always has an
// Input.GitilesCommit. This ensures that this build and any child builds
// always use a consistent HEAD.
//
// The host and project for the GitilesCommit are determined by the
// `ResolveRepo` strategy, with the fallback remote being either the default
// integration remote set in this file, or the override set by the
// `recipe_integration_remote` property.
func ensureGitilesCommit(ctx context.Context, build *buildbucketpb.Build) error {
	if build.Input.GitilesCommit != nil {
		return nil
	}
	fallbackRemote, err := props.IntegrationRemote(build)
	if err != nil {
		return fmt.Errorf("failed to resolve recipe_integration_remote property: %v", err)
	}
	if fallbackRemote == "" {
		fallbackRemote = defaultIntegrationRemote
	}
	host, project, err := rbc.ResolveRepo(build.Input, fallbackRemote)
	if err != nil {
		return fmt.Errorf("failed to resolve host and project: %v", err)
	}
	revision, err := resolveHead(ctx, host, project)
	if err != nil {
		return fmt.Errorf("failed to resolve HEAD: %v", err)
	}
	build.Input.GitilesCommit = &buildbucketpb.GitilesCommit{
		Host:    host,
		Project: project,
		Id:      revision,
	}
	return nil
}

// resolveRecipeVersion resolves the recipe version to use.
//
// If Build.Input.GerritChanges indicates that the recipes repo is under test,
// we short-circuit immediately.
//
// In all other cases, we ensure that the recipe version property is set in
// Build.Input.Properties.
func resolveRecipeVersion(ctx context.Context, build *buildbucketpb.Build) error {
	recipeVersion, err := props.RecipeVersion(build)
	if err != nil {
		return err
	}
	// If the recipe version property is already set, we're done.
	if recipeVersion != "" {
		return nil
	}
	// If the recipes repo is under test, we're done. checkoutRecipes will use
	// the build input directly.
	hasRecipeChange, err := rbc.HasRepoChange(build.Input, recipesRemote)
	if err != nil {
		return fmt.Errorf("could not determine whether build input has recipe change: %v", err)
	}
	if hasRecipeChange {
		return nil
	}
	// Otherwise, resolve the recipe version from the checkout.
	integrationURL, err := rbc.ResolveIntegrationURL(build.Input)
	if err != nil {
		return err
	}
	// Do a checkout in a temporary directory to avoid checkout conflicts with
	// the recipe's working directory.
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}
	dir, err := ioutil.TempDir(cwd, "checkout")
	if err != nil {
		return err
	}
	defer os.RemoveAll(dir)
	tctx, cancel := context.WithTimeout(ctx, defaultCheckoutTimeout)
	defer cancel()
	if err := checkout.Checkout(tctx, *build.Input, *integrationURL, defaultIntegrationRef, dir); err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}
	manifestXML, err := os.Open(filepath.Join(dir, recipesManifest))
	if err != nil {
		return err
	}
	defer manifestXML.Close()
	proj, err := manifest.ResolveRecipesProject(manifestXML, recipesRemote)
	if err != nil {
		return err
	}
	return props.SetRecipeVersion(build, proj.Revision)
}

// checkoutRecipes checks out the recipes.git repo in dir.
//
// If the build input has a recipe change, then checkout directly using the
// build input.
//
// Otherwise, checkout at the revision specified by Build.Input.Properties.
func checkoutRecipes(ctx context.Context, dir string, build *buildbucketpb.Build) error {
	recipesURL, err := url.Parse(recipesRemote)
	if err != nil {
		return fmt.Errorf("could not parse URL %s", recipesRemote)
	}
	var input *buildbucketpb.Build_Input
	hasRecipeChange, err := rbc.HasRepoChange(build.Input, recipesRemote)
	if err != nil {
		return fmt.Errorf("could not determine whether build input has recipe change: %v", err)
	}
	if hasRecipeChange {
		input = build.Input
	} else {
		recipeVersion, err := props.RecipeVersion(build)
		if err != nil {
			return err
		}
		if recipeVersion == "" {
			return errors.New("recipe_version property is unexpectedly empty")
		}
		input = &buildbucketpb.Build_Input{
			GitilesCommit: &buildbucketpb.GitilesCommit{
				Host:    recipesURL.Host,
				Project: strings.TrimLeft(recipesURL.Path, "/"),
				Id:      recipeVersion,
			},
		}
	}
	tctx, cancel := context.WithTimeout(ctx, defaultCheckoutTimeout)
	defer cancel()
	if err := checkout.Checkout(tctx, *input, *recipesURL, "", dir); err != nil {
		return fmt.Errorf("failed to checkout: %v", err)
	}
	return nil
}

// executeRecipe exec's `recipes.py luciexe`, passing build.proto through stdin.
func executeRecipe(ctx context.Context, dir string, build *buildbucketpb.Build) error {
	cmd := exec.CommandContext(ctx, path.Join(dir, "recipes.py"), "luciexe")
	inputData, err := proto.Marshal(build)
	if err != nil {
		return fmt.Errorf("could not marshal input build: %v", err)
	}
	cmd.Stdin = bytes.NewBuffer(inputData)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// fail logs an error and returns exit code 1.
func fail(ctx context.Context, reason string) {
	logging.Errorf(ctx, reason)
	os.Exit(1)
}

func main() {
	ctx := context.Background()
	ctx = gologger.StdConfig.Use(ctx)
	data, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		fail(ctx, fmt.Sprintf("failed to read build.proto from stdin: %v", err))
	}
	build := &buildbucketpb.Build{}
	if err := proto.Unmarshal(data, build); err != nil {
		fail(ctx, fmt.Sprintf("failed to unmarshal build.proto from stdin: %v", err))
	}
	if err := ensureGitilesCommit(ctx, build); err != nil {
		fail(ctx, fmt.Sprintf("failed to ensure that build has a gitiles commit: %v", err))
	}
	if err := resolveRecipeVersion(ctx, build); err != nil {
		fail(ctx, fmt.Sprintf("failed to resolve recipe version: %v", err))
	}
	// Do recipes checkout in a temporary directory to avoid checkout conflicts
	// with the recipe's working directory.
	cwd, err := os.Getwd()
	if err != nil {
		fail(ctx, fmt.Sprintf("failed to get working directory: %v", err))
	}
	dir, err := ioutil.TempDir(cwd, "recipes-checkout")
	if err != nil {
		fail(ctx, fmt.Sprintf("failed to get create tempdir: %v", err))
	}
	defer os.RemoveAll(dir)
	if err := checkoutRecipes(ctx, dir, build); err != nil {
		fail(ctx, fmt.Sprintf("failed to download recipe bundle: %v", err))
	}
	os.Stderr.Sync()
	if err := executeRecipe(ctx, dir, build); err != nil {
		fail(ctx, fmt.Sprintf("failed to execute recipe: %v", err))
	}
}