tools/integration/fint/set.go

// Copyright 2021 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package fint

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	fintpb "go.fuchsia.dev/fuchsia/tools/integration/fint/proto"
	"go.fuchsia.dev/fuchsia/tools/lib/hostplatform"
	"go.fuchsia.dev/fuchsia/tools/lib/isatty"
	"go.fuchsia.dev/fuchsia/tools/lib/logger"
	"go.fuchsia.dev/fuchsia/tools/lib/osmisc"
	"go.fuchsia.dev/fuchsia/tools/lib/subprocess"
)

var (
	// Path within a checkout to script which will run a hermetic Python interpreter.
	vendoredPythonScriptPath = []string{"scripts", "fuchsia-vendored-python"}

	// Path within a checkout to script which will clobber a build when new fences appear.
	forceCleanScript = []string{"build", "force_clean", "force_clean_if_needed.py"}
)

// Set runs `gn gen` given a static and context spec. It's intended to be
// consumed as a library function.
func Set(ctx context.Context, staticSpec *fintpb.Static, contextSpec *fintpb.Context, skipLocalArgs bool) (*fintpb.SetArtifacts, error) {
	platform, err := hostplatform.Name()
	if err != nil {
		return nil, err
	}

	// TODO move to setImpl, add unit tests
	if err := forceCleanIfNeeded(ctx, contextSpec, platform); err != nil {
		return nil, err
	}

	artifacts, err := setImpl(ctx, &subprocess.Runner{}, staticSpec, contextSpec, platform, skipLocalArgs)
	if err != nil && artifacts != nil && artifacts.FailureSummary == "" {
		// Fall back to using the error text as the failure summary if the
		// failure summary is unset. It's better than failing without emitting
		// any information.
		artifacts.FailureSummary = err.Error()
	}
	return artifacts, err
}

// setImpl runs `gn gen` along with any post-processing steps, and returns a
// SetArtifacts object containing metadata produced by GN and post-processing.
func setImpl(
	ctx context.Context,
	runner subprocessRunner,
	staticSpec *fintpb.Static,
	contextSpec *fintpb.Context,
	platform string,
	skipLocalArgs bool,
) (*fintpb.SetArtifacts, error) {
	if contextSpec.CheckoutDir == "" {
		return nil, fmt.Errorf("checkout_dir must be set")
	}
	if contextSpec.BuildDir == "" {
		return nil, fmt.Errorf("build_dir must be set")
	}

	genArgs, err := genArgs(ctx, staticSpec, contextSpec, skipLocalArgs)
	if err != nil {
		return nil, err
	}

	artifacts := &fintpb.SetArtifacts{
		UseGoma: staticSpec.UseGoma,
		Metadata: &fintpb.SetArtifacts_Metadata{
			Board:      staticSpec.Board,
			Optimize:   strings.ToLower(staticSpec.Optimize.String()),
			Product:    staticSpec.Product,
			TargetArch: strings.ToLower(staticSpec.TargetArch.String()),
			Variants:   staticSpec.Variants,
		},
		// True if any toolchain is using RBE and needs reproxy to run.
		// Note: bazel+RBE doesn't require reproxy.
		EnableRbe: staticSpec.RustRbeEnable || staticSpec.CxxRbeEnable || staticSpec.LinkRbeEnable,
	}

	if contextSpec.ArtifactDir != "" {
		artifacts.GnTracePath = filepath.Join(contextSpec.ArtifactDir, "gn_trace.json")
	}
	genStdout, err := runGen(ctx, runner, staticSpec, contextSpec, platform, artifacts.GnTracePath, genArgs)
	if err != nil {
		artifacts.FailureSummary = genStdout
		return artifacts, err
	}

	// Only run build graph analysis if the result will be emitted via
	// artifacts, and if we actually care about checking the result.
	if contextSpec.ArtifactDir != "" && staticSpec.SkipIfUnaffected {
		var changedFiles []string
		for _, f := range contextSpec.ChangedFiles {
			changedFiles = append(changedFiles, f.Path)
		}
		sb, err := shouldBuild(ctx, runner, contextSpec.BuildDir, contextSpec.CheckoutDir, platform, changedFiles)
		if err != nil {
			return artifacts, err
		}
		artifacts.SkipBuild = !sb
	}
	return artifacts, err
}

// forceCleanIfNeeded clobbers the build dir if new clean build fences are found, see
// //build/force_clean/README.md for details.
func forceCleanIfNeeded(ctx context.Context, contextSpec *fintpb.Context, platform string) (err error) {
	if _, err := os.Stat(contextSpec.BuildDir); os.IsNotExist(err) {
		// no need to clean anything if there's nothing there
		return nil
	}
	scriptRunner := &subprocess.Runner{}
	scriptRunner.Dir = contextSpec.CheckoutDir
	return scriptRunner.Run(ctx, []string{
		filepath.Join(append([]string{contextSpec.CheckoutDir}, vendoredPythonScriptPath...)...),
		filepath.Join(append([]string{contextSpec.CheckoutDir}, forceCleanScript...)...),
		"--gn-bin",
		thirdPartyPrebuilt(contextSpec.CheckoutDir, platform, "gn"),
		"--checkout-dir",
		contextSpec.CheckoutDir,
		"--build-dir",
		contextSpec.BuildDir,
	}, subprocess.RunOptions{})
}

func runGen(
	ctx context.Context,
	runner subprocessRunner,
	staticSpec *fintpb.Static,
	contextSpec *fintpb.Context,
	platform string,
	gnTracePath string,
	args []string,
) (genStdout string, err error) {
	gn := thirdPartyPrebuilt(contextSpec.CheckoutDir, platform, "gn")

	formattedArgs := gnFormat(ctx, gn, runner, args)
	logger.Infof(ctx, "GN args:\n%s\n", formattedArgs)

	// gn will return an error if the argument list is too long, so write the
	// args directly to the build dir instead of using the --args flag.
	if f, err := osmisc.CreateFile(filepath.Join(contextSpec.BuildDir, "args.gn")); err != nil {
		return "", fmt.Errorf("failed to create args.gn: %w", err)
	} else if _, err := io.WriteString(f, formattedArgs); err != nil {
		return "", fmt.Errorf("failed to write args.gn: %w", err)
	}

	genCmd := []string{
		gn,
		"gen",
		contextSpec.BuildDir,
		fmt.Sprintf("--root=%s", contextSpec.CheckoutDir),
		"--check=system",
		"--fail-on-unused-args",
		// If --ninja-executable is set, GN runs `ninja -t restat build.ninja`
		// after generating the ninja files, updating the cached modified
		// timestamps of files. This avoids extra regens when running ninja
		// repeatedly under some circumstances.
		fmt.Sprintf("--ninja-executable=%s", thirdPartyPrebuilt(contextSpec.CheckoutDir, platform, "ninja")),
	}

	if isatty.IsTerminal() {
		genCmd = append(genCmd, "--color")
	}
	if gnTracePath != "" {
		genCmd = append(genCmd, fmt.Sprintf("--tracelog=%s", gnTracePath))
	}
	if staticSpec.ExportRustProject {
		genCmd = append(genCmd, "--export-rust-project")
	}
	for _, f := range staticSpec.IdeFiles {
		genCmd = append(genCmd, fmt.Sprintf("--ide=%s", f))
	}
	for _, s := range staticSpec.JsonIdeScripts {
		genCmd = append(genCmd, fmt.Sprintf("--json-ide-script=%s", s))
	}

	// Always generate the ninja_outputs.json file used by //build/api/client
	genCmd = append(genCmd, "--ninja-outputs-file=ninja_outputs.json")

	// When `gn gen` fails, it outputs a brief helpful error message to stdout.
	var stdoutBuf bytes.Buffer
	if err := runner.Run(ctx, genCmd, subprocess.RunOptions{Stdout: io.MultiWriter(&stdoutBuf, os.Stdout)}); err != nil {
		return stdoutBuf.String(), fmt.Errorf("error running gn gen: %w", err)
	}
	return stdoutBuf.String(), nil
}

// findGNIFile returns the relative path to a board or product file in a
// checkout, given a basename. It checks the root of the checkout as well as
// each vendor/* directory for a file matching "<dirname>/<basename>.gni", e.g.
// "boards/core.gni".
func findGNIFile(checkoutDir, dirname, basename string) (string, error) {
	dirs, err := filepath.Glob(filepath.Join(checkoutDir, "vendor", "*", dirname))
	if err != nil {
		return "", err
	}
	dirs = append(dirs, filepath.Join(checkoutDir, dirname))

	for _, dir := range dirs {
		path := filepath.Join(dir, fmt.Sprintf("%s.gni", basename))
		exists, err := osmisc.FileExists(path)
		if err != nil {
			return "", err
		}
		if exists {
			return filepath.Rel(checkoutDir, path)
		}
	}

	return "", nil
}

func genArgs(ctx context.Context, staticSpec *fintpb.Static, contextSpec *fintpb.Context, skipLocalArgs bool) ([]string, error) {
	// GN variables to set via args (mapping from variable name to value).
	vars := make(map[string]interface{})
	// GN list variables that could historically be set by products, and should go
	// in their own block in args.gn.  Append or assign is controlled when these
	// are formatted as lines.
	targetLists := make(map[string][]string)
	// GN targets to import.
	var imports []string
	// GN vars for managing tests.
	testVars := make(map[string]interface{})

	if staticSpec.TargetArch == fintpb.Static_ARCH_UNSPECIFIED {
		// Board files declare `target_cpu` so it's not necessary to set
		// `target_cpu` as long as we have a board file.
		if staticSpec.Board == "" {
			return nil, fmt.Errorf("target_arch must be set if board is not")
		}
	} else {
		vars["target_cpu"] = strings.ToLower(staticSpec.TargetArch.String())
	}

	if staticSpec.Optimize == fintpb.Static_OPTIMIZE_UNSPECIFIED {
		return nil, fmt.Errorf("optimize is unspecified or invalid")
	}
	vars["is_debug"] = staticSpec.Optimize == fintpb.Static_DEBUG

	if contextSpec.ClangToolchainDir != "" {
		if staticSpec.UseGoma {
			return nil, fmt.Errorf("goma is not supported for builds using a custom clang toolchain")
		}
		vars["clang_prefix"] = filepath.Join(contextSpec.ClangToolchainDir, "bin")
	}
	if contextSpec.GccToolchainDir != "" {
		if staticSpec.UseGoma {
			return nil, fmt.Errorf("goma is not supported for builds using a custom gcc toolchain")
		}
		vars["gcc_tool_dir"] = filepath.Join(contextSpec.GccToolchainDir, "bin")
	}
	if contextSpec.RustToolchainDir != "" {
		vars["rustc_prefix"] = filepath.Join(contextSpec.RustToolchainDir)
	}

	vars["use_goma"] = staticSpec.UseGoma
	vars["rust_rbe_enable"] = staticSpec.RustRbeEnable
	vars["cxx_rbe_enable"] = staticSpec.CxxRbeEnable
	vars["link_rbe_enable"] = staticSpec.LinkRbeEnable
	vars["enable_bazel_remote_rbe"] = staticSpec.BazelRbeEnable

	if staticSpec.Product != "" {
		basename := filepath.Base(staticSpec.Product)
		vars["build_info_product"] = strings.Split(basename, ".")[0]
		imports = append(imports, staticSpec.Product)
	}

	if staticSpec.Board != "" {
		basename := filepath.Base(staticSpec.Board)
		vars["build_info_board"] = strings.Split(basename, ".")[0]
		imports = append(imports, staticSpec.Board)
	}

	if contextSpec.SdkId != "" {
		vars["sdk_id"] = contextSpec.SdkId
	}

	if contextSpec.ReleaseVersion != "" {
		vars["build_info_version"] = contextSpec.ReleaseVersion
	}

	if staticSpec.TestDurationsFile != "" {
		testDurationsFile := staticSpec.TestDurationsFile
		exists, err := osmisc.FileExists(filepath.Join(contextSpec.CheckoutDir, testDurationsFile))
		if err != nil {
			return nil, fmt.Errorf("failed to check if TestDurationsFile exists: %w", err)
		}
		if !exists {
			testDurationsFile = staticSpec.DefaultTestDurationsFile
		}
		vars["test_durations_file"] = testDurationsFile
	}

	// TODO(ihuh): Remove once builders are including this target in their universe
	// packages.
	if staticSpec.IncludeZbiTests {
		staticSpec.UniversePackages = append(staticSpec.UniversePackages, "//bundles/boot_tests")
	}

	for varName, values := range map[string][]string{
		"base_package_labels":     staticSpec.BasePackages,
		"cache_package_labels":    staticSpec.CachePackages,
		"universe_package_labels": staticSpec.UniversePackages,
	} {
		targetLists[varName] = values
	}

	// These list variables are never initialized by a product, so they can be
	// directly set.
	vars["host_labels"] = staticSpec.HostLabels
	testVars["hermetic_test_package_labels"] = staticSpec.HermeticTestPackages
	testVars["test_package_labels"] = staticSpec.TestPackages
	testVars["e2e_test_labels"] = staticSpec.E2ETestLabels
	testVars["host_test_labels"] = staticSpec.HostTestLabels

	if len(staticSpec.Variants) != 0 {
		vars["select_variant"] = staticSpec.Variants
	}
	if contextSpec.CollectCoverage && len(contextSpec.ChangedFiles) > 0 {
		var profileSourceFiles []string
		for _, file := range contextSpec.ChangedFiles {
			profileSourceFiles = append(profileSourceFiles, fmt.Sprintf("//%s", file.Path))
		}
		// Profile changed files, and only changed files.
		vars["profile_source_files"] = profileSourceFiles
		vars["dont_profile_source_files"] = []string{}
	}

	if contextSpec.PgoProfilePath != "" {
		vars["pgo_profile_path"] = filepath.Join(contextSpec.PgoProfilePath)
	}

	if staticSpec.EnableGoCache {
		vars["gocache_dir"] = filepath.Join(contextSpec.CacheDir, "go_cache")
	} else if staticSpec.UseTemporaryGoCache {
		// We wish to have the go cache directory be deterministic,
		// because the cache directory winds up in various ninja action
		// commandlines, so having the cache directory change between
		// builds means that those action's outputs are dirtied, and we
		// re-run actions on incremental builds that differ only in go
		// cache directory.
		// However, we do still wish to preserve the invariant that the
		// cache directory is empty when UseTemporaryGoCache is
		// requested.  Thus, instead of generating a random directory
		// with os.TempDir(), we generate a predictable, deterministic
		// path to serve as the go cache directory, and ensure that it
		// is empty at the time of `fint set`, which assuming no racing
		// work is occurring on the same machine (which is a safe
		// assumption in our build infra), is semantically equivalent
		// while allowing better build caching.
		dir := filepath.Join(os.TempDir(), "fuchsia_go_cache")
		if err := os.RemoveAll(dir); err != nil {
			return nil, err
		}
		if err := os.MkdirAll(dir, 0700); err != nil {
			return nil, err
		}
		vars["gocache_dir"] = dir
	}

	if staticSpec.EnableRustCache {
		vars["rust_incremental"] = filepath.Join(contextSpec.CacheDir, "rust_cache")
	}

	var importArgs, varArgs, targetListArgs, testArgs, localArgs []string

	// Add comments to make args.gn more readable.
	varArgs = append(varArgs, "\n\n# Basic args:")
	targetListArgs = append(targetListArgs, "\n\n# Target lists:")
	testArgs = append(testArgs, "\n\n# Tests to add to build: (these are validated by test-type)")

	// vars are directly set in the "basic args" block
	for k, v := range vars {
		varArgs = append(varArgs, fmt.Sprintf("%s=%s", k, toGNValue(v)))
	}

	for _, arg := range staticSpec.GnArgs {
		if strings.HasPrefix(arg, "import(") {
			importArgs = append(importArgs, arg)
		} else if strings.Contains(arg, "+=") {
			// any list-append operation is almost always trying to change the target
			// lists set by a product or board, so add it to that block:
			targetListArgs = append(targetListArgs, arg)
		} else {
			// any directly-assigned args are more likely general build arguments.
			varArgs = append(varArgs, arg)
		}
	}
	sort.Strings(varArgs)

	for k, v := range targetLists {
		// Products and Boards are now using their own namespace of GN args, and not
		// using these target lists, which are used only by infra or developers.
		targetListArgs = append(targetListArgs, fmt.Sprintf("%s=%s", k, toGNValue(v)))
	}
	sort.Strings(targetListArgs)

	// Add the "build_only_labels" to the end of appendArgs so that it stays in
	// the "#Target lists:" block, which is semantically where it belongs.
	targetListArgs = append(targetListArgs, fmt.Sprintf("%s=%s", "build_only_labels", toGNValue(staticSpec.BuildOnlyLabels)))

	// The test vars are kept in a particular order to match the BUILD.gn files.
	for _, k := range []string{"hermetic_test_package_labels", "test_package_labels", "e2e_test_labels", "host_test_labels"} {
		testArgs = append(testArgs, fmt.Sprintf("%s=%s", k, toGNValue(testVars[k])))
	}

	if len(staticSpec.DeveloperTestLabels) != 0 && skipLocalArgs {
		return nil, fmt.Errorf("'developer_test_labels' cannot be provided when 'skipLocalArgs' is true")
	}
	testArgs = append(testArgs, "\n\n# Additional tests: (not validated by test-type)")
	testArgs = append(testArgs, fmt.Sprintf("%s=%s", "developer_test_labels", toGNValue(staticSpec.DeveloperTestLabels)))

	for _, p := range imports {
		importArgs = append(importArgs, fmt.Sprintf(`import("//%s")`, p))
	}
	sort.Strings(importArgs)

	if !skipLocalArgs {
		localArgsPath := filepath.Join(contextSpec.CheckoutDir, "local/args.gn")
		localArgsBytes, err := os.ReadFile(localArgsPath)
		localArgsContents := string(localArgsBytes)
		if err == nil {
			logger.Infof(ctx, "Including local args from %s.", localArgsPath)
			localArgs = append(localArgs, fmt.Sprintf("\n\n# Local args from %s:", localArgsPath))
			localArgs = append(localArgs, localArgsContents)
		} else if errors.Is(err, os.ErrNotExist) {
			logger.Infof(ctx, "No local args available.")
		} else {
			return nil, err
		}
	}

	// Ensure that imports come before args that set or modify variables, as
	// otherwise the imported files might blindly redefine variables set or
	// modified by other arguments.
	var finalArgs []string
	finalArgs = append(finalArgs, importArgs...)
	finalArgs = append(finalArgs, varArgs...)
	finalArgs = append(finalArgs, targetListArgs...)
	finalArgs = append(finalArgs, testArgs...)
	finalArgs = append(finalArgs, localArgs...)
	finalArgs = append(finalArgs, "\n")
	return finalArgs, nil
}

// gnFormat makes a best-effort attempt to format the input arguments using `gn
// format` so that the output will be more readable in case of any errors like
// unknown variables. If formatting fails (e.g. due to a syntax error) we'll
// just return the unformatted args and let `gn gen` return an error; otherwise
// we'd need duplicated error handling code to handle both syntax errors from
// `gn format` and non-syntax errors from `gn gen`.
func gnFormat(ctx context.Context, gn string, runner subprocessRunner, args []string) string {
	unformatted := strings.Join(args, "\n")
	var output bytes.Buffer
	opts := subprocess.RunOptions{
		Stdout: &output,
		Stderr: io.Discard,
		Stdin:  strings.NewReader(unformatted),
	}
	if err := runner.Run(ctx, []string{gn, "format", "--stdin"}, opts); err != nil {
		return unformatted
	}
	return output.String()
}

// toGNValue converts a Go value to a string representation of the corresponding
// GN value by inspecting the Go value's type. This makes the logic to set GN
// args more readable and less error-prone, with no need for patterns like
// fmt.Sprintf(`"%s"`) repeated everywhere.
//
// For example:
// - toGNValue(true) => `true`
// - toGNValue("foo") => `"foo"` (a string containing literal double-quotes)
// - toGNValue([]string{"foo", "bar"}) => `["foo","bar"]`
func toGNValue(x interface{}) string {
	switch val := x.(type) {
	case bool:
		return fmt.Sprintf("%v", val)
	case string:
		// Apply double-quotes to strings, but not to GN scopes like
		// {variant="asan-fuzzer" target_type=["fuzzed_executable"]}
		if strings.HasPrefix(val, "{") && strings.HasSuffix(val, "}") {
			return val
		}
		return fmt.Sprintf(`"%s"`, val)
	case []string:
		var values []string
		for _, element := range val {
			values = append(values, toGNValue(element))
		}
		return fmt.Sprintf("[%s]", strings.Join(values, ","))
	default:
		panic(fmt.Sprintf("unsupported arg value type %T", val))
	}
}