tools/botanist/target/qemu.go

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package target

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/creack/pty"

	"go.fuchsia.dev/fuchsia/tools/bootserver/lib"
	"go.fuchsia.dev/fuchsia/tools/botanist/lib"
	"go.fuchsia.dev/fuchsia/tools/lib/iomisc"
	"go.fuchsia.dev/fuchsia/tools/lib/logger"
	"go.fuchsia.dev/fuchsia/tools/lib/osmisc"
	"go.fuchsia.dev/fuchsia/tools/lib/ring"
	"go.fuchsia.dev/fuchsia/tools/qemu"
)

const (
	// qemuSystemPrefix is the prefix of the QEMU binary name, which is of the
	// form qemu-system-<QEMU arch suffix>.
	qemuSystemPrefix = "qemu-system"

	// DefaultInterfaceName is the name given to the emulated tap interface.
	defaultInterfaceName = "qemu"

	// Default networking values.
	defaultMACAddr       = "52:54:00:63:5e:7a"
	defaultLinkLocalAddr = "fe80::5054:ff:fe63:5e7a"
	defaultNodename      = "step-atom-yard-juicy"

	// The size in bytes of minimimum desired size for the storage-full image.
	// The image should be large enough to hold all downloaded test packages
	// for a given test shard.
	//
	// No host-side disk blocks are allocated on extension (by use of the `fvm`
	// host tool), so the operation is cheap regardless of the size we extend to.
	storageFullMinSize int64 = 10000000000 // 10Gb
)

// qemuTargetMapping maps the Fuchsia target name to the name recognized by QEMU.
var qemuTargetMapping = map[string]qemu.Target{
	"x64":   qemu.TargetEnum.X86_64,
	"arm64": qemu.TargetEnum.AArch64,
}

// MinFS is the configuration for the MinFS filesystem image.
type MinFS struct {
	// Image is the path to the filesystem image.
	Image string `json:"image"`

	// PCIAddress is the PCI address to map the device at.
	PCIAddress string `json:"pci_address"`
}

// QEMUConfig is a QEMU configuration.
type QEMUConfig struct {
	// Path is a path to a directory that contains QEMU system binary.
	Path string `json:"path"`

	// Target is the QEMU target to emulate.
	Target string `json:"target"`

	// CPU is the number of processors to emulate.
	CPU int `json:"cpu"`

	// Memory is the amount of memory (in MB) to provide.
	Memory int `json:"memory"`

	// KVM specifies whether to enable hardware virtualization acceleration.
	KVM bool `json:"kvm"`

	// Serial gives whether to create a 'serial device' for the QEMU instance.
	// This option should be used judiciously, as it can slow the process down.
	Serial bool `json:"serial"`

	// Whether User networking is enabled; if false, a Tap interface will be used.
	UserNetworking bool `json:"user_networking"`

	// MinFS is the filesystem to mount as a device.
	MinFS *MinFS `json:"minfs,omitempty"`

	// Path to the fvm host tool.
	FVMTool string `json:"fvm_tool"`
}

// QEMUTarget is a QEMU target.
type QEMUTarget struct {
	config  QEMUConfig
	opts    Options
	c       chan error
	process *os.Process
	serial  io.ReadWriteCloser
	pts     *os.File
}

// NewQEMUTarget returns a new QEMU target with a given configuration.
func NewQEMUTarget(config QEMUConfig, opts Options) (*QEMUTarget, error) {
	var serial io.ReadWriteCloser
	var pts *os.File
	if config.Serial {
		// We can run QEMU 'in a terminal' by creating a pseudoterminal slave and
		// attaching it as the process' std(in|out|err) streams. Running it in a
		// terminal - and redirecting serial to stdio - allows us to use the
		// associated pseudoterminal master as the 'serial device' for the
		// instance.
		var ptm *os.File
		var err error
		ptm, pts, err = pty.Open()
		if err != nil {
			return nil, fmt.Errorf("failed to create ptm/pts pair: %w", err)
		}

		// We should be streaming serial's output to stdout even if nothing is
		// actively reading from it.
		stdoutBuf := ring.NewBuffer(botanist.SerialLogBufferSize)
		go io.Copy(io.MultiWriter(stdoutBuf, os.Stdout), ptm)
		serial = struct {
			io.Reader
			io.WriteCloser
		}{stdoutBuf, ptm}
	}

	// TODO(joshuaseaton): Figure out how to manage ownership of pts so that it
	// may be closed.
	return &QEMUTarget{
		config: config,
		opts:   opts,
		c:      make(chan error),
		serial: serial,
		pts:    pts,
	}, nil
}

// Nodename returns the name of the target node.
func (t *QEMUTarget) Nodename() string {
	return defaultNodename
}

// IPv6Addr returns the global unicast IPv6 address of the qemu instance.
func (t *QEMUTarget) IPv6Addr() string {
	return fmt.Sprintf("%s%%%s", defaultLinkLocalAddr, defaultInterfaceName)
}

// IPv4Addr returns a nil address, as DHCP is not currently configured.
func (t *QEMUTarget) IPv4Addr() (net.IP, error) {
	return nil, nil
}

// Serial returns the serial device associated with the target for serial i/o.
func (t *QEMUTarget) Serial() io.ReadWriteCloser {
	return t.serial
}

// SSHKey returns the private SSH key path associated with a previously embedded authorized key.
func (t *QEMUTarget) SSHKey() string {
	return t.opts.SSHKey
}

// Start starts the QEMU target.
func (t *QEMUTarget) Start(ctx context.Context, images []bootserver.Image, args []string) (err error) {
	// We create a working directory for the QEMU process below, but cannot
	// clean it up until we error or until the process finishes. The former is
	// handled in this block, while the latter is handled in a goroutine below.
	var workdir string
	go func() {
		if workdir != "" && err != nil {
			os.RemoveAll(workdir)
		}
	}()

	if t.process != nil {
		return fmt.Errorf("a process has already been started with PID %d", t.process.Pid)
	}
	qemuCmd := &qemu.QEMUCommandBuilder{}

	qemuTarget, ok := qemuTargetMapping[t.config.Target]
	if !ok {
		return fmt.Errorf("invalid target %q", t.config.Target)
	}
	qemuCmd.SetTarget(qemuTarget, t.config.KVM)

	if t.config.Path == "" {
		return fmt.Errorf("directory must be set")
	}
	qemuSystem := filepath.Join(t.config.Path, fmt.Sprintf("%s-%s", qemuSystemPrefix, qemuTarget))
	absQEMUSystemPath, err := normalizeFile(qemuSystem)
	if err != nil {
		return fmt.Errorf("could not find qemu binary %q: %w", qemuSystem, err)
	}
	qemuCmd.SetBinary(absQEMUSystemPath)

	var qemuKernel, zirconA, storageFull bootserver.Image
	for _, img := range images {
		switch img.Name {
		case "kernel_qemu-kernel":
			qemuKernel = img
		case "zbi_zircon-a":
			zirconA = img
		case "blk_storage-full":
			storageFull = img
		}
	}
	if qemuKernel.Reader == nil {
		return fmt.Errorf("could not find kernel_qemu-kernel")
	}
	if zirconA.Reader == nil {
		return fmt.Errorf("could not find zbi_zircon-a")
	}
	// The QEMU command needs to be invoked within an emptm directory, as QEMU
	// will attempt to pick up files from its working directory, one notable
	// culprit being multiboot.bin.  This can result in strange behavior.
	workdir, err = ioutil.TempDir("", "qemu-working-dir")
	if err != nil {
		return err
	}

	if err := copyImagesToDir(ctx, workdir, &qemuKernel, &zirconA, &storageFull); err != nil {
		return err
	}

	// Now that the images hav successfully been copied to the working
	// directory, Path points to their path on disk.
	qemuCmd.SetKernel(qemuKernel.Path)
	qemuCmd.SetInitrd(zirconA.Path)

	if storageFull.Path != "" {
		if t.config.FVMTool != "" {
			if err := extendStorageFull(ctx, &storageFull, t.config.FVMTool, storageFullMinSize); err != nil {
				return fmt.Errorf("failed to extend fvm.blk to %d bytes: %v", storageFullMinSize, err)
			}
		}
		qemuCmd.AddVirtioBlkPciDrive(qemu.Drive{
			ID:   "maindisk",
			File: storageFull.Path,
		})
	}

	if t.config.MinFS != nil {
		absMinFsPath, err := normalizeFile(t.config.MinFS.Image)
		if err != nil {
			return fmt.Errorf("could not find minfs image %q: %v", t.config.MinFS.Image, err)
		}
		// Swarming hard-links Isolate downloads with a cache and the very same
		// cached minfs image will be used across multiple tasks. To ensure
		// that it remains blank, we must break its link.
		if err := overwriteFileWithCopy(absMinFsPath); err != nil {
			return err
		}
		qemuCmd.AddVirtioBlkPciDrive(qemu.Drive{
			ID:   "testdisk",
			File: absMinFsPath,
			Addr: t.config.MinFS.PCIAddress,
		})
	}

	netdev := qemu.Netdev{
		ID:  "net0",
		MAC: defaultMACAddr,
	}
	if t.config.UserNetworking {
		netdev.User = &qemu.NetdevUser{}
	} else {
		netdev.Tap = &qemu.NetdevTap{
			Name: defaultInterfaceName,
		}
	}
	qemuCmd.AddNetwork(netdev)

	// The system will halt on a kernel panic instead of rebooting.
	qemuCmd.AddKernelArg("kernel.halt-on-panic=true")
	// Print a message if `dm poweroff` times out.
	qemuCmd.AddKernelArg("devmgr.suspend-timeout-debug=true")
	// Do not print colors.
	qemuCmd.AddKernelArg("TERM=dumb")
	if t.config.Target == "x64" {
		// Necessary to redirect to stdout.
		qemuCmd.AddKernelArg("kernel.serial=legacy")
	}
	for _, arg := range args {
		qemuCmd.AddKernelArg(arg)
	}

	qemuCmd.SetCPUCount(t.config.CPU)
	qemuCmd.SetMemory(t.config.Memory)
	qemuCmd.SetFlag("-nographic")
	qemuCmd.SetFlag("-serial", "stdio")
	qemuCmd.SetFlag("-monitor", "none")

	invocation, err := qemuCmd.Build()
	if err != nil {
		return err
	}

	cmd := &exec.Cmd{
		Path:   invocation[0],
		Args:   invocation,
		Dir:    workdir,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	if t.pts != nil {
		cmd.Stdin = t.pts
		cmd.Stdout = t.pts
		cmd.Stderr = t.pts
		cmd.SysProcAttr = &syscall.SysProcAttr{
			Setctty: true,
			Setsid:  true,
			Ctty:    int(t.pts.Fd()),
		}
	}
	logger.Debugf(ctx, "QEMU invocation:\n%s", strings.Join(invocation, " "))

	if err := cmd.Start(); err != nil {
		return fmt.Errorf("failed to start: %w", err)
	}
	t.process = cmd.Process

	go func() {
		err := cmd.Wait()
		if err != nil {
			err = fmt.Errorf("QEMU invocation error: %w", err)
		}
		t.c <- err
		os.RemoveAll(workdir)
	}()
	return nil
}

// Restart stops the QEMU target and starts it again.
func (t *QEMUTarget) Restart(context.Context) error {
	return ErrUnimplemented
}

// Stop stops the QEMU target.
func (t *QEMUTarget) Stop(context.Context) error {
	if t.process == nil {
		return fmt.Errorf("QEMU target has not yet been started")
	}
	err := t.process.Kill()
	t.process = nil
	return err
}

// Wait waits for the QEMU target to stop.
func (t *QEMUTarget) Wait(ctx context.Context) error {
	return <-t.c
}

func copyImagesToDir(ctx context.Context, dir string, imgs ...*bootserver.Image) error {
	// Copy each in a goroutine for efficiency's sake.
	errs := make(chan error, len(imgs))
	var wg sync.WaitGroup
	wg.Add(len(imgs))
	for _, img := range imgs {
		go func(img *bootserver.Image) {
			if img.Reader != nil {
				if err := copyImageToDir(ctx, dir, img); err != nil {
					errs <- err
				}
			}
			wg.Done()
		}(img)
	}
	wg.Wait()
	select {
	case err := <-errs:
		return err
	default:
		return nil
	}
}

func copyImageToDir(ctx context.Context, dir string, img *bootserver.Image) error {
	dest := filepath.Join(dir, img.Name)

	f, ok := img.Reader.(*os.File)
	if ok {
		if err := osmisc.CopyFile(f.Name(), dest); err != nil {
			return err
		}
		img.Path = dest
		return nil
	}

	f, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer f.Close()

	// Log progress to avoid hitting I/O timeout in case of slow transfers.
	ticker := time.NewTicker(30 * time.Second)
	defer ticker.Stop()
	go func() {
		for range ticker.C {
			logger.Debugf(ctx, "transferring %s...\n", img.Name)
		}
	}()

	if _, err := io.Copy(f, iomisc.ReaderAtToReader(img.Reader)); err != nil {
		return fmt.Errorf("failed to copy image %q to %q: %w", img.Name, dest, err)
	}
	img.Path = dest

	// We no longer need the reader at this point.
	c, ok := img.Reader.(io.Closer)
	if ok {
		c.Close()
	}
	img.Reader = nil
	return nil
}

func normalizeFile(path string) (string, error) {
	if _, err := os.Stat(path); err != nil {
		return "", err
	}
	absPath, err := filepath.Abs(path)
	if err != nil {
		return "", err
	}
	return absPath, nil
}

func overwriteFileWithCopy(path string) error {
	tmpfile, err := ioutil.TempFile(filepath.Dir(path), "botanist")
	if err != nil {
		return err
	}
	defer tmpfile.Close()
	if err := osmisc.CopyFile(path, tmpfile.Name()); err != nil {
		return err
	}
	return os.Rename(tmpfile.Name(), path)
}

func extendStorageFull(ctx context.Context, storageFull *bootserver.Image, fvmTool string, size int64) error {
	if storageFull.Size >= size {
		return nil
	}
	absToolPath, err := filepath.Abs(fvmTool)
	if err != nil {
		return err
	}
	logger.Debugf(ctx, "extending fvm.blk to %d bytes", size)
	cmd := exec.CommandContext(ctx, absToolPath, storageFull.Path, "extend", "--length", strconv.Itoa(int(size)))
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	storageFull.Size = size
	return nil
}