tools/botanist/cmd/run.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"go.fuchsia.dev/fuchsia/tools/botanist"
	"go.fuchsia.dev/fuchsia/tools/botanist/constants"
	"go.fuchsia.dev/fuchsia/tools/botanist/targets"
	"go.fuchsia.dev/fuchsia/tools/lib/environment"
	"go.fuchsia.dev/fuchsia/tools/lib/ffxutil"
	"go.fuchsia.dev/fuchsia/tools/lib/flagmisc"
	"go.fuchsia.dev/fuchsia/tools/lib/logger"
	"go.fuchsia.dev/fuchsia/tools/lib/osmisc"
	"go.fuchsia.dev/fuchsia/tools/lib/serial"
	"go.fuchsia.dev/fuchsia/tools/lib/subprocess"
	"go.fuchsia.dev/fuchsia/tools/lib/syslog"
	"go.fuchsia.dev/fuchsia/tools/testing/testrunner"
	testrunnerconstants "go.fuchsia.dev/fuchsia/tools/testing/testrunner/constants"

	"github.com/google/subcommands"
	"golang.org/x/sync/errgroup"
)

// RunCommand is a Command implementation for booting a device and running a
// given command locally.
type RunCommand struct {
	// ConfigFile is the path to the target configurations.
	configFile string

	// DownloadManifest is the path we should write the package server's
	// download manifest to.
	downloadManifest string

	// ImageManifest is a path to an image manifest.
	imageManifest string

	// ProductBundle is a path to product_bundles.json file.
	productBundles string

	// ProductBundleName is a name of product bundle getting used.
	productBundleName string

	// IsBootTest tells whether the product bundle provided is for a boot test.
	isBootTest bool

	// Netboot tells botanist to netboot (and not to pave).
	netboot bool

	// ZirconArgs are kernel command-line arguments to pass on boot.
	zirconArgs flagmisc.StringsValue

	// Timeout is the duration allowed for the command to finish execution.
	timeout time.Duration

	// syslogDir, if nonempty, is the directory in which system syslogs will be written.
	syslogDir string

	// SshKey is the path to a private SSH user key.
	sshKey string

	// serialLogDir, if nonempty, is the directory in which system serial logs will be written.
	serialLogDir string

	// RepoURL specifies the URL of a package repository.
	repoURL string

	// BlobURL optionally specifies the URL of where a package repository's blobs may be served from.
	// Defaults to $repoURL/blobs.
	blobURL string

	// localRepo specifies the path to a local package repository. If set,
	// botanist will spin up a package server to serve packages from this
	// repository.
	localRepo string

	// The path to the ffx tool.
	ffxPath string

	// The level of experimental ffx features to enable.
	//
	// The following levels enable the following ffx features:
	// 0 or greater: ffx target flash, ffx bootloader boot, CSO-only mode
	// 1 or greater: ffx emu, ffx log
	// 2 or greater: ffx test, ffx target snapshot, keeps ffx output dir for debugging
	// 3: enables parallel test execution
	ffxExperimentLevel int

	// When true skips setting up the targets.
	skipSetup bool

	// Args passed to testrunner
	testrunnerOptions testrunner.Options

	// When true, upload to resultdb from testrunner.
	uploadToResultDB bool

	// The timeout to wait for an SSH connection after booting the target.
	bootupTimeout time.Duration
}

func (*RunCommand) Name() string {
	return "run"
}

func (*RunCommand) Usage() string {
	return `
botanist run [flags...] tests-file

flags:
`
}

func (*RunCommand) Synopsis() string {
	return fmt.Sprintf("boots a device and executes all tests found in the JSON [tests-file].")
}

func (r *RunCommand) SetFlags(f *flag.FlagSet) {
	f.StringVar(&r.configFile, "config", "", "path to file of device config")
	f.StringVar(&r.imageManifest, "images", "", "path to an image manifest")
	f.StringVar(&r.productBundles, "product-bundles", "", "path to product_bundles.json file")
	f.StringVar(&r.productBundleName, "product-bundle-name", "", "name of product bundle to use")
	f.BoolVar(&r.isBootTest, "boot-test", false, "whether the provided product bundle is for a boot test.")
	f.BoolVar(&r.netboot, "netboot", false, "if set, botanist will not pave; but will netboot instead")
	f.Var(&r.zirconArgs, "zircon-args", "kernel command-line arguments")
	f.DurationVar(&r.timeout, "timeout", 0, "duration allowed for the command to finish execution, a value of 0 (zero) will not impose a timeout.")
	f.StringVar(&r.syslogDir, "syslog-dir", "", "the directory to write all system logs to.")
	f.StringVar(&r.sshKey, "ssh", "", "file containing a private SSH user key; if not provided, a private key will be generated.")
	f.StringVar(&r.serialLogDir, "serial-log-dir", "", "the directory to write all serial logs to.")
	f.StringVar(&r.repoURL, "repo", "", "URL at which to configure a package repository; if the placeholder of \"localhost\" will be resolved and scoped as appropriate")
	f.StringVar(&r.blobURL, "blobs", "", "URL at which to serve a package repository's blobs; if the placeholder of \"localhost\" will be resolved and scoped as appropriate")
	f.StringVar(&r.localRepo, "local-repo", "", "path to a local package repository; the repo and blobs flags are ignored when this is set")
	f.StringVar(&r.ffxPath, "ffx", "", "Path to the ffx tool.")
	f.StringVar(&r.downloadManifest, "download-manifest", "", "Path to a manifest containing all package server downloads")
	f.IntVar(&r.ffxExperimentLevel, "ffx-experiment-level", 0, "The level of experimental features to enable. If -ffx is not set, this will have no effect.")
	f.BoolVar(&r.skipSetup, "skip-setup", false, "if set, botanist will not set up a target.")
	// Temporary flag to enable a soft transition to uploading test results from botanist rather than from the recipe.
	f.BoolVar(&r.uploadToResultDB, "upload-to-resultdb", false, "if set, test results will be uploaded to ResultDB from testrunner.")
	f.DurationVar(&r.bootupTimeout, "bootup-timeout", 0, "duration allowed for the command to finish execution, a value of 0 (zero) will fall back to the default.")

	// Parsing of testrunner options.
	f.StringVar(&r.testrunnerOptions.OutDir, "out-dir", "", "Optional path where a directory containing test results should be created.")
	f.StringVar(&r.testrunnerOptions.NsjailPath, "nsjail", "", "Optional path to an NsJail binary to use for linux host test sandboxing.")
	f.StringVar(&r.testrunnerOptions.NsjailRoot, "nsjail-root", "", "Path to the directory to use as the NsJail root directory")
	f.StringVar(&r.testrunnerOptions.LocalWD, "C", "", "Working directory of local testing subprocesses; if unset the current working directory will be used.")
	f.StringVar(&r.testrunnerOptions.SnapshotFile, "snapshot-output", "", "The output filename for the snapshot. This will be created in the output directory.")
	f.BoolVar(&r.testrunnerOptions.PrefetchPackages, "prefetch-packages", false, "Prefetch any test packages in the background.")
	f.BoolVar(&r.testrunnerOptions.UseSerial, "use-serial", false, "Use serial to run tests on the target.")
}

func (r *RunCommand) setupFFX(ctx context.Context, fuchsiaTargets []targets.FuchsiaTarget, primaryTarget targets.FuchsiaTarget) (func(), error) {
	var cleanup func()
	if r.ffxPath == "" {
		return cleanup, fmt.Errorf("ffx path must be provided with the -ffx flag.")
	}
	ffxOutputsDir := filepath.Join(os.Getenv(testrunnerconstants.TestOutDirEnvKey), "ffx_outputs")

	extraConfigs := ffxutil.ConfigSettings{
		Level: "global",
		Settings: map[string]any{
			"daemon.autostart":       false,
			"discovery.mdns.enabled": false,
		},
	}
	ffx, err := ffxutil.NewFFXInstance(ctx, r.ffxPath, "", []string{}, primaryTarget.Nodename(), primaryTarget.SSHKey(), ffxOutputsDir, extraConfigs)
	if err != nil {
		return cleanup, err
	}
	stdout, stderr, flush := botanist.NewStdioWriters(ctx, "ffx")
	defer flush()
	ffx.SetStdoutStderr(stdout, stderr)
	if r.ffxExperimentLevel > 0 {
		if err := ffx.SetLogLevel(ctx, ffxutil.Debug); err != nil {
			return cleanup, err
		}
	}
	if err := ffx.Run(ctx, "config", "env"); err != nil {
		return cleanup, err
	}

	cmd := ffx.Command("daemon", "start")
	daemonLog, err := osmisc.CreateFile(filepath.Join(ffxOutputsDir, "daemon.log"))
	if err != nil {
		return cleanup, err
	}
	cmd.Stdout = daemonLog
	logger.Debugf(ctx, "%s", cmd.Args)
	// Use a new context so that the subprocess can only be terminated by
	// a direct call to the cancel function.
	daemonCtx, daemonCancel := context.WithCancel(context.Background())
	if err := cmd.Start(); err != nil {
		return cleanup, err
	}
	// Wait for the daemon process to terminate in a separate goroutine
	// and log when it finishes in order to detect if the process gets
	// terminated earlier than expected.
	cmdWait := make(chan error)
	go func() {
		// Using subprocess.WaitForCmd() instead of cmd.Wait() ensures that
		// the function returns when the context is done.
		if err := subprocess.WaitForCmd(daemonCtx, cmd); err != nil {
			logger.Errorf(ctx, "daemon process finished with err: %s", err)
		} else {
			logger.Debugf(ctx, "ffx daemon process finished")
		}
		cmdWait <- err
	}()
	cleanup = func() {
		// TODO(https://fxbug.dev/42071857): Clean up daemon by sending a SIGTERM to the
		// process once that is supported.
		if err := ffx.Stop(); err != nil {
			logger.Errorf(ctx, "failed to stop ffx daemon: %s", err)
		}

		// Wait for the daemon process to finish before closing the log.
		daemonCancel()
		<-cmdWait
		if err := daemonLog.Close(); err != nil {
			logger.Errorf(ctx, "failed to close ffx daemon log: %s", err)
		}
	}

	for _, t := range fuchsiaTargets {
		// Start serial servers for all targets. Will no-op for targets that
		// already have serial servers.
		if err := t.StartSerialServer(); err != nil {
			return cleanup, err
		}
		// Attach an ffx instance for all targets. All ffx instances will use the same
		// config and daemon, but run commands against its own specified target.
		ffxForTarget := ffxutil.FFXWithTarget(ffx, t.Nodename())
		t.SetFFX(&targets.FFXInstance{ffxForTarget, r.ffxExperimentLevel}, ffx.Env())
	}

	return cleanup, ffx.WaitForDaemon(ctx)
}

func (r *RunCommand) setupSerialLog(ctx context.Context, eg *errgroup.Group, fuchsiaTargets []targets.FuchsiaTarget) error {
	if r.serialLogDir == "" {
		return nil
	}

	if err := os.Mkdir(r.serialLogDir, os.ModePerm); err != nil {
		return err
	}

	for _, t := range fuchsiaTargets {
		t := t
		eg.Go(func() error {
			logger.Debugf(ctx, "starting serial collection for target %s", t.Nodename())

			// Create a new file to capture the serial log for this nodename.
			serialLogName := fmt.Sprintf("%s_serial_log.txt", t.Nodename())
			// TODO(https://fxbug.dev/42150891): Remove once there are no dependencies on this filename.
			if len(fuchsiaTargets) == 1 {
				serialLogName = "serial_log.txt"
			}
			serialLogPath := filepath.Join(r.serialLogDir, serialLogName)
			absPath, err := filepath.Abs(serialLogPath)
			if err != nil {
				return fmt.Errorf("failed to get abspath of serial log: %w", err)
			}
			if err := os.Setenv(constants.SerialLogEnvKey, absPath); err != nil {
				logger.Debugf(ctx, "failed to set %s to %s", constants.SerialLogEnvKey, absPath)
			}

			// Start capturing the serial log for this target.
			if err := t.CaptureSerialLog(serialLogPath); err != nil && ctx.Err() == nil {
				return err
			}
			return nil
		})
	}
	return nil
}

func (r *RunCommand) setupPackageServer(ctx context.Context) (*botanist.PackageServer, error) {
	if r.localRepo == "" {
		return nil, nil
	}

	var port int
	pkgSrvPort := os.Getenv(constants.PkgSrvPortKey)
	if pkgSrvPort == "" {
		logger.Warningf(ctx, "%s is empty, using default port %d", constants.PkgSrvPortKey, botanist.DefaultPkgSrvPort)
		port = botanist.DefaultPkgSrvPort
	} else {
		var err error
		port, err = strconv.Atoi(pkgSrvPort)
		if err != nil {
			return nil, err
		}
	}

	pkgSrv, err := botanist.NewPackageServer(ctx, r.localRepo, r.repoURL, r.blobURL, r.downloadManifest, port)
	if err != nil {
		return pkgSrv, err
	}
	// TODO(rudymathu): Once gcsproxy and remote package serving are deprecated, remove
	// the repoURL and blobURL from the command line flags.
	r.repoURL = pkgSrv.RepoURL
	r.blobURL = pkgSrv.BlobURL

	return pkgSrv, nil
}

func (r *RunCommand) dispatchTests(ctx context.Context, cancel context.CancelFunc, eg *errgroup.Group, baseTargets []targets.Base, fuchsiaTargets []targets.FuchsiaTarget, primaryTarget targets.FuchsiaTarget, testsPath string) {
	// Disable usb mass storage to determine if it affects NUC stability.
	// TODO(rudymathu): Remove this once stability is achieved.
	r.zirconArgs = append(r.zirconArgs, "driver.usb_mass_storage.disable")

	// TODO(https://fxbug.dev/42169595#c74): Remove this once CDC-ether flakiness
	// has been resolved.
	r.zirconArgs = append(r.zirconArgs, "driver.usb_cdc.log=debug")

	// Log any failures after running tests.
	for _, t := range fuchsiaTargets {
		t := t
		eg.Go(func() error {
			if err := t.Wait(ctx); err != nil && err != targets.ErrUnimplemented && ctx.Err() == nil {
				return fmt.Errorf("target %s failed: %w", t.Nodename(), err)
			}
			return nil
		})
	}

	// Dispatch tests.
	eg.Go(func() error {
		// Signal other goroutines to exit when tests complete.
		defer cancel()

		if r.productBundles == "" {
			return fmt.Errorf("-product-bundles is required")
		}
		if r.productBundleName == "" {
			return fmt.Errorf("-product-bundle-name is required")
		}
		startOpts := targets.StartOptions{
			Netboot:           r.netboot,
			ImageManifest:     r.imageManifest,
			ZirconArgs:        r.zirconArgs,
			ProductBundles:    r.productBundles,
			ProductBundleName: r.productBundleName,
			IsBootTest:        r.isBootTest,
			BootupTimeout:     r.bootupTimeout,
		}

		if err := targets.StartTargets(ctx, startOpts, fuchsiaTargets); err != nil {
			return fmt.Errorf("%s: %w", constants.FailedToStartTargetMsg, err)
		}
		logger.Debugf(ctx, "successfully started all targets")

		defer func() {
			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
			defer cancel()
			targets.StopTargets(ctx, fuchsiaTargets)
		}()

		// Create a testbed config file. We have to do this after starting the
		// targets so that we can get their IP addresses.
		testbedConfig, err := r.createTestbedConfig(baseTargets)
		if err != nil {
			return err
		}
		defer os.Remove(testbedConfig)

		if !r.netboot {
			for _, t := range fuchsiaTargets {
				t := t
				client, err := t.SSHClient()
				if err != nil {
					if err := r.dumpSyslogOverSerial(ctx, t.SerialSocketPath()); err != nil {
						logger.Errorf(ctx, err.Error())
					}
					return err
				}
				if r.repoURL != "" {
					if err := t.AddPackageRepository(client, r.repoURL, r.blobURL); err != nil {
						return err
					}
					logger.Debugf(ctx, "added package repo to target %s", t.Nodename())
				}
				if r.syslogDir != "" {
					if _, err := os.Stat(r.syslogDir); errors.Is(err, os.ErrNotExist) {
						if err := os.Mkdir(r.syslogDir, os.ModePerm); err != nil {
							return err
						}
					}
					go func() {
						syslogName := fmt.Sprintf("%s_syslog.txt", t.Nodename())
						// TODO(https://fxbug.dev/42150891): Remove when there are no dependencies on this filename.
						if len(fuchsiaTargets) == 1 {
							syslogName = "syslog.txt"
						}
						syslogPath := filepath.Join(r.syslogDir, syslogName)
						if err := t.CaptureSyslog(client, syslogPath, r.repoURL, r.blobURL); err != nil && ctx.Err() == nil {
							logger.Errorf(ctx, "%s at %s: %s", constants.FailedToCaptureSyslogMsg, syslogPath, err)
						}
					}()
				}
			}
		}
		err = r.runAgainstTarget(ctx, primaryTarget, testsPath, testbedConfig)
		// Cancel ctx to notify other goroutines that this routine has completed.
		// If another goroutine gets an error and the context is canceled, it
		// should return nil so that we always prioritize the result from this
		// goroutine.
		cancel()
		return err
	})
}

func (r *RunCommand) execute(ctx context.Context, args []string) error {
	ctx, cancel := context.WithCancel(ctx)
	if r.timeout != 0 {
		ctx, cancel = context.WithTimeout(ctx, r.timeout)
	}

	go func() {
		<-ctx.Done()
		// Log the timeout for tefmocheck to detect it.
		if ctx.Err() == context.DeadlineExceeded {
			logger.Errorf(ctx, "%s (%s)", constants.CommandExceededTimeoutMsg, r.timeout)
		}
	}()
	defer cancel()

	testsPath := args[0]

	if r.skipSetup {
		if err := testrunner.SetupAndExecute(ctx, r.testrunnerOptions, testsPath); err != nil {
			return fmt.Errorf("testrunner with flags: %v, with timeout: %s, failed: %w", r.testrunnerOptions, r.timeout, err)
		}
		return nil
	}

	// Parse targets out from the target configuration file.
	baseTargets, fuchsiaTargets, err := r.deriveTargetsFromFile(ctx)
	if err != nil {
		return err
	}
	// Determine the target that a command will be run against and logs will be
	// streamed from.
	primaryTarget := fuchsiaTargets[0]

	cleanup, err := r.setupFFX(ctx, fuchsiaTargets, primaryTarget)
	if cleanup != nil {
		defer cleanup()
	}
	if err != nil {
		return err
	}

	eg, ctx := errgroup.WithContext(ctx)

	if err := r.setupSerialLog(ctx, eg, fuchsiaTargets); err != nil {
		return err
	}

	// Run any preflights to prepare the testbed.
	if err := r.runPreflights(ctx); err != nil {
		return err
	}

	pkgSrv, err := r.setupPackageServer(ctx)
	if pkgSrv != nil {
		defer pkgSrv.Close()
	}
	if err != nil {
		return err
	}

	r.dispatchTests(ctx, cancel, eg, baseTargets, fuchsiaTargets, primaryTarget, testsPath)

	if err := eg.Wait(); err != nil {
		return err
	}

	return nil
}

// runPreflights runs opaque preflight commands passed to botanist from
// the calling infrastructure.
func (r *RunCommand) runPreflights(ctx context.Context) error {
	logger.Debugf(ctx, "checking for preflights")
	botfilePath := os.Getenv("SWARMING_BOT_FILE")
	if botfilePath == "" {
		return nil
	}
	data, err := os.ReadFile(botfilePath)
	if err != nil {
		return err
	}
	if len(data) == 0 {
		// There were no commands in the botfile, exit out.
		return nil
	}
	type preflightCommands struct {
		Commands [][]string `json:"commands"`
	}
	var cmds preflightCommands
	if err := json.Unmarshal(data, &cmds); err != nil {
		return err
	}
	runner := subprocess.Runner{
		Env: os.Environ(),
	}
	for _, c := range cmds.Commands {
		logger.Debugf(ctx, "running preflight %s", c)
		if err := runner.Run(ctx, c, subprocess.RunOptions{}); err != nil {
			return err
		}
	}
	if len(cmds.Commands) > 0 {
		// Some preflight commands can cause side effects that take up to 30s.
		time.Sleep(30 * time.Second)
	}
	logger.Debugf(ctx, "done running preflights")
	return nil
}

// createTestbedConfig creates a configuration file that describes the targets
// attached and returns the path to the file.
func (r *RunCommand) createTestbedConfig(baseTargets []targets.Base) (string, error) {
	var testbedConfig []any
	for _, t := range baseTargets {
		c, err := t.TestConfig(r.netboot)
		if err != nil {
			return "", err
		}
		testbedConfig = append(testbedConfig, c)
	}

	data, err := json.Marshal(testbedConfig)
	if err != nil {
		return "", err
	}

	f, err := os.CreateTemp("", "testbed_config")
	if err != nil {
		return "", err
	}
	defer f.Close()
	if _, err := f.Write(data); err != nil {
		return "", err
	}
	return f.Name(), nil
}

// dumpSyslogOverSerial runs log_listener over serial to collect logs that may
// help with debugging. This is intended to be used when SSH connection fails to
// get some information about the failure mode prior to exiting.
func (r *RunCommand) dumpSyslogOverSerial(ctx context.Context, socketPath string) error {
	socket, err := serial.NewSocket(ctx, socketPath)
	if err != nil {
		return fmt.Errorf("newSerialSocket failed: %w", err)
	}
	defer socket.Close()
	if err := serial.RunDiagnostics(ctx, socket); err != nil {
		return fmt.Errorf("failed to run serial diagnostics: %w", err)
	}
	// Dump the existing syslog buffer. This may not work if pkg-resolver is not
	// up yet, in which case it will just print nothing.
	cmds := []serial.Command{
		{Cmd: syslog.LogListenerWithArgs("--dump_logs", "yes"), SleepDuration: 5 * time.Second},
	}
	if err := serial.RunCommands(ctx, socket, cmds); err != nil {
		return fmt.Errorf("failed to dump syslog over serial: %w", err)
	}
	return nil
}

func (r *RunCommand) runAgainstTarget(ctx context.Context, t targets.FuchsiaTarget, testsPath string, testbedConfig string) error {
	testrunnerEnv := map[string]string{
		constants.NodenameEnvKey:      t.Nodename(),
		constants.SerialSocketEnvKey:  t.SerialSocketPath(),
		constants.ECCableEnvKey:       os.Getenv(constants.ECCableEnvKey),
		constants.TestbedConfigEnvKey: testbedConfig,
	}

	// If |netboot| is true, then we assume that fuchsia is not provisioned
	// with a netstack; in this case, do not try to establish a connection.
	if !r.netboot {
		var addr net.IPAddr
		ipv6, err := t.IPv6()
		if err != nil {
			return err
		}
		if ipv6 != nil {
			addr = *ipv6
		}
		ipv4, err := t.IPv4()
		if err != nil {
			return err
		}
		if ipv4 != nil {
			addr.IP = ipv4
			addr.Zone = ""
		}

		testrunnerEnv[constants.DeviceAddrEnvKey] = addr.String()
		testrunnerEnv[constants.IPv4AddrEnvKey] = ipv4.String()
		testrunnerEnv[constants.IPv6AddrEnvKey] = ipv6.String()
		// Add the target address in order to skip MDNS discovery.
		if err := t.GetFFX().Run(ctx, "target", "add", addr.String()); err != nil {
			return err
		}
	}

	// One would assume this should only be provisioned when paving, but
	// there are some tests that attempt to SSH into a netbooted image that
	// has our SSH keys baked into it. Therefore, we add the SSH key to the
	// environment unconditionally. Additionally, some tools like FFX often
	// require the SSH key path to be absolute (https://fxbug.dev/42051867).
	if t.SSHKey() != "" {
		absKeyPath, err := filepath.Abs(t.SSHKey())
		if err != nil {
			return err
		}
		testrunnerEnv[constants.SSHKeyEnvKey] = absKeyPath
	}

	// TODO(https://fxbug.dev/42063235): testrunner does heavy use of env
	// variables. Setting these env variables is temporary until we refactor
	// testrunner to take these variables as arguments or flags.
	for k, v := range testrunnerEnv {
		err := os.Setenv(k, v)
		if err != nil {
			return fmt.Errorf("error setting env variable %s=%s. %w", k, v, err)
		}
	}
	setEnviron(t.FFXEnv())
	r.testrunnerOptions.FFX = t.GetFFX().FFXInstance
	r.testrunnerOptions.FFXExperimentLevel = r.ffxExperimentLevel

	if err := testrunner.SetupAndExecute(ctx, r.testrunnerOptions, testsPath); err != nil {
		return fmt.Errorf("testrunner with flags: %v, with timeout: %s, failed: %w", r.testrunnerOptions, r.timeout, err)
	}
	return nil
}

// setEnviron sets |environ| into the os.Env.
// The string in the environ slice must be in the format "key=value".
func setEnviron(environ []string) {
	for _, env := range environ {
		keyval := strings.Split(env, "=")
		os.Setenv(keyval[0], keyval[1])
	}
}

func (r *RunCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	args := f.Args()
	if len(args) == 0 {
		return subcommands.ExitUsageError
	}

	// If the TestOutDirEnvKey was set, that means botanist is being run in an infra
	// setting and thus needs an isolated environment.
	testOutDir, needsIsolatedEnv := os.LookupEnv(testrunnerconstants.TestOutDirEnvKey)
	cleanUp, err := environment.Ensure(needsIsolatedEnv)
	if err != nil {
		logger.Errorf(ctx, "failed to setup environment: %s", err)
		return subcommands.ExitFailure
	}
	defer cleanUp()

	if needsIsolatedEnv {
		// Use a temp directory for the output directory which we will move to the
		// actual testOutDir once the command completes. Otherwise, when run in a
		// swarming task, a subprocess that doesn't properly finish could still be
		// writing to the out dir as we try to upload the contents with the swarming
		// task outputs which will result in the swarming bot failing with BOT_DIED.
		tmpOutDir, err := os.MkdirTemp("", "")
		if err != nil {
			return subcommands.ExitFailure
		}
		if err := os.Setenv(testrunnerconstants.TestOutDirEnvKey, tmpOutDir); err != nil {
			return subcommands.ExitFailure
		}
		defer func() {
			if skippedFiles, err := osmisc.CopyDir(tmpOutDir, testOutDir, osmisc.SkipUnknownFiles); err != nil {
				logger.Errorf(ctx, "failed to copy outputs to %s: %s", testOutDir, err)
				// TODO(https://fxbug.dev/42079078): If we fail to copy outputs, at least copy
				// the ffx logs over so we can debug. Remove when attached bug is
				// fixed.
				if r.ffxPath != "" {
					ffxLogsDir := filepath.Join("ffx_outputs", "ffx_logs")
					if _, err := os.Stat(filepath.Join(testOutDir, ffxLogsDir)); os.IsNotExist(err) {
						if _, err := osmisc.CopyDir(filepath.Join(tmpOutDir, ffxLogsDir), filepath.Join(testOutDir, ffxLogsDir), osmisc.RaiseError); err != nil {
							logger.Errorf(ctx, "failed to copy ffx logs to %s: %s", filepath.Join(testOutDir, ffxLogsDir), err)
						}
					}
				}
			} else if len(skippedFiles) > 0 {
				skippedFilesTxt := filepath.Join(testOutDir, "skipped_files.txt")
				if err := os.WriteFile(skippedFilesTxt, []byte(strings.Join(skippedFiles, "\n")), os.ModePerm); err != nil {
					logger.Errorf(ctx, "failed to write %s: %s\nskipped files: %s", skippedFilesTxt, err, skippedFiles)
				}
			}

			if err := os.Setenv(testrunnerconstants.TestOutDirEnvKey, testOutDir); err != nil {
				logger.Errorf(ctx, "failed to reset %s to %s: %s", testrunnerconstants.TestOutDirEnvKey, testOutDir, err)
			}
			if err := os.RemoveAll(tmpOutDir); err != nil {
				logger.Errorf(ctx, "failed to remove temp outputs dir %s: %s", tmpOutDir, err)
			}
		}()
	}

	r.blobURL = os.ExpandEnv(r.blobURL)
	r.repoURL = os.ExpandEnv(r.repoURL)
	if err := r.execute(ctx, args); err != nil {
		logger.Errorf(ctx, "%s", err)
		return subcommands.ExitFailure
	}
	return subcommands.ExitSuccess
}

func (r *RunCommand) deriveTargetsFromFile(ctx context.Context) ([]targets.Base, []targets.FuchsiaTarget, error) {
	data, err := os.ReadFile(r.configFile)
	if err != nil {
		return nil, nil, fmt.Errorf("%s: %w", constants.ReadConfigFileErrorMsg, err)
	}
	var configs []json.RawMessage
	if err := json.Unmarshal(data, &configs); err != nil {
		return nil, nil, fmt.Errorf("could not unmarshal config file as a JSON list: %w", err)
	}

	var baseTargets []targets.Base
	var fuchsiaTargets []targets.FuchsiaTarget

	for _, config := range configs {
		t, err := targets.FromJSON(ctx, config, targets.Options{
			Netboot: r.netboot,
			SSHKey:  r.sshKey,
		})
		if err != nil {
			return nil, nil, err
		}
		baseTargets = append(baseTargets, t)
		if f, ok := t.(targets.FuchsiaTarget); ok {
			fuchsiaTargets = append(fuchsiaTargets, f)
		}
	}

	if len(fuchsiaTargets) == 0 {
		return nil, nil, fmt.Errorf("no Fuchsia targets found")
	}

	return baseTargets, fuchsiaTargets, nil
}