[botanistd] Factor out QEMU invocation
This change refactors fuchsia.googlesource.com/tools/qemu and
fuchsia.googlesource.com/tools/cmd/botanist/qemu. Now, the former is a
generic package that can be used to construct QEMU invocation without
any Fuchsia-specific logic. The Fuchsia specific logic has been moved to
fuchsia.googlesource.com/tools/botanist/target so it can be reused in
different implementations. The command is a simple frontend that just
passes through arguments.
This change also introduces the Target interface. Currently, QEMUTarget is
the only implementation of this interface, but subsequent changes should
also introduce DeviceTarget implementation that will provide the same
interface on top of Zedboot. We might also introduce GCETarget once the
GCE support is ready. This will allow controlling arbitrary devices from
commands like `botanist run`, completely replacing `botanist qemu` and
`botanist zedboot`.
Change-Id: I279a207fb39a47bde426089b1683477d9936922c
diff --git a/botanist/target/qemu.go b/botanist/target/qemu.go
new file mode 100644
index 0000000..817465d
--- /dev/null
+++ b/botanist/target/qemu.go
@@ -0,0 +1,250 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package target
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+
+ "fuchsia.googlesource.com/tools/build"
+ "fuchsia.googlesource.com/tools/qemu"
+)
+
+const (
+ // qemuSystemPrefix is the prefix of the QEMU binary name, which is of the
+ // form qemu-system-<QEMU arch suffix>.
+ qemuSystemPrefix = "qemu-system"
+)
+
+// qemuTargetMapping maps the Fuchsia target name to the name recognized by QEMU.
+var qemuTargetMapping = map[string]string{
+ "x64": qemu.TargetX86_64,
+ "arm64": qemu.TargetAArch64,
+}
+
+// MinFS is the configuration for the MinFS filesystem image.
+type MinFS struct {
+ // Image is the path to the filesystem image.
+ Image string `json:"image"`
+
+ // PCIAddress is the PCI address to map the device at.
+ PCIAddress string `json:"pci_address"`
+}
+
+// QEMUConfig is a QEMU configuration.
+type QEMUConfig struct {
+ // Path is a path to a directory that contains QEMU system binary.
+ Path string `json:"path"`
+
+ // Target is the QEMU target to emulate.
+ Target string `json:"target"`
+
+ // CPU is the number of processors to emulate.
+ CPU int `json:"cpu"`
+
+ // Memory is the amount of memory (in MB) to provide.
+ Memory int `json:"memory"`
+
+ // KVM specifies whether to enable hardware virtualization acceleration.
+ KVM bool `json:"kvm"`
+
+ // Network specifies whether to emulate a network device.
+ Network bool `json:"network"`
+
+ // MinFS is the filesystem to mount as a device.
+ MinFS *MinFS `json:"minfs,omitempty"`
+}
+
+// NewQEMUConfig returns a new QEMU configuration.
+func NewQEMUConfig() *QEMUConfig {
+ return &QEMUConfig{
+ CPU: 4,
+ Memory: 4096,
+ }
+}
+
+// QEMUTarget is a QEMU target.
+type QEMUTarget struct {
+ config QEMUConfig
+
+ c chan error
+
+ cmd *exec.Cmd
+ status error
+}
+
+// NewQEMUTarget returns a new QEMU target with a given configuration.
+func NewQEMUTarget(config QEMUConfig) *QEMUTarget {
+ return &QEMUTarget{
+ config: config,
+ c: make(chan error),
+ }
+}
+
+// Start starts the QEMU target.
+func (d *QEMUTarget) Start(ctx context.Context, images build.Images, args []string) error {
+ qemuTarget, ok := qemuTargetMapping[d.config.Target]
+ if !ok {
+ return fmt.Errorf("invalid target %q", d.config.Target)
+ }
+
+ if d.config.Path == "" {
+ return fmt.Errorf("directory must be set")
+ }
+ qemuSystem := filepath.Join(d.config.Path, fmt.Sprintf("%s-%s", qemuSystemPrefix, qemuTarget))
+ if _, err := os.Stat(qemuSystem); err != nil {
+ return fmt.Errorf("could not find qemu-system binary %q: %v", qemuSystem, err)
+ }
+
+ qemuKernel := images.Get("qemu-kernel")
+ if qemuKernel == nil {
+ return fmt.Errorf("could not find qemu-kernel")
+ }
+ zirconA := images.Get("zircon-a")
+ if zirconA == nil {
+ return fmt.Errorf("could not find zircon-a")
+ }
+
+ var drives []qemu.Drive
+ if storageFull := images.Get("storage-full"); storageFull != nil {
+ drives = append(drives, qemu.Drive{
+ ID: "maindisk",
+ File: storageFull.Path,
+ })
+ }
+ if d.config.MinFS != nil {
+ if _, err := os.Stat(d.config.MinFS.Image); err != nil {
+ return fmt.Errorf("could not find minfs image %q: %v", d.config.MinFS.Image, err)
+ }
+ file, err := filepath.Abs(d.config.MinFS.Image)
+ if err != nil {
+ return err
+ }
+ // Swarming hard-links Isolate downloads with a cache and the very same
+ // cached minfs image will be used across multiple tasks. To ensure
+ // that it remains blank, we must break its link.
+ if err := overwriteFileWithCopy(file); err != nil {
+ return err
+ }
+ drives = append(drives, qemu.Drive{
+ ID: "testdisk",
+ File: file,
+ Addr: d.config.MinFS.PCIAddress,
+ })
+ }
+
+ var networks []qemu.Netdev
+ if d.config.Network {
+ networks = append(networks, qemu.Netdev{
+ ID: "net0",
+ })
+ }
+
+ config := qemu.Config{
+ Binary: qemuSystem,
+ Target: qemuTarget,
+ CPU: d.config.CPU,
+ Memory: d.config.Memory,
+ KVM: d.config.KVM,
+ Kernel: qemuKernel.Path,
+ Initrd: zirconA.Path,
+ Drives: drives,
+ Networks: networks,
+ }
+
+ // The system will halt on a kernel panic instead of rebooting.
+ args = append(args, "kernel.halt-on-panic=true")
+ // Print a message if `dm poweroff` times out.
+ args = append(args, "devmgr.suspend-timeout-debug=true")
+ // Do not print colors.
+ args = append(args, "TERM=dumb")
+ if d.config.Target == "x64" {
+ // Necessary to redirect to stdout.
+ args = append(args, "kernel.serial=legacy")
+ }
+
+ invocation, err := qemu.CreateInvocation(config, args)
+ if err != nil {
+ return err
+ }
+
+ // The QEMU command needs to be invoked within an empty directory, as QEMU
+ // will attempt to pick up files from its working directory, one notable
+ // culprit being multiboot.bin. This can result in strange behavior.
+ workdir, err := ioutil.TempDir("", "qemu-working-dir")
+ if err != nil {
+ return err
+ }
+
+ d.cmd = &exec.Cmd{
+ Path: invocation[0],
+ Args: invocation,
+ Dir: workdir,
+ Stdout: os.Stdout,
+ Stderr: os.Stderr,
+ }
+ log.Printf("QEMU invocation:\n%s", invocation)
+
+ if err := d.cmd.Start(); err != nil {
+ os.RemoveAll(workdir)
+ return fmt.Errorf("failed to start: %v", err)
+ }
+
+ // Ensure that the working directory when QEMU finishes whether the Wait
+ // method is invoked or not.
+ go func() {
+ defer os.RemoveAll(workdir)
+ d.c <- qemu.CheckExitCode(d.cmd.Wait())
+ }()
+
+ return nil
+}
+
+// Stop stops the QEMU target.
+func (d *QEMUTarget) Stop(ctx context.Context) error {
+ return d.cmd.Process.Kill()
+}
+
+// Wait waits for the QEMU target to stop.
+func (d *QEMUTarget) Wait(ctx context.Context) error {
+ return <-d.c
+}
+
+func overwriteFileWithCopy(path string) error {
+ tmpfile, err := ioutil.TempFile(filepath.Dir(path), "botanist")
+ if err != nil {
+ return err
+ }
+ defer tmpfile.Close()
+ if err := copyFile(path, tmpfile.Name()); err != nil {
+ return err
+ }
+ return os.Rename(tmpfile.Name(), path)
+}
+
+func copyFile(src, dest string) error {
+ in, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+ info, err := in.Stat()
+ if err != nil {
+ return err
+ }
+ out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, info.Mode().Perm())
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+ _, err = io.Copy(out, in)
+ return err
+}
diff --git a/cmd/botanist/qemu.go b/cmd/botanist/qemu.go
index 9be78db..67aaec8 100644
--- a/cmd/botanist/qemu.go
+++ b/cmd/botanist/qemu.go
@@ -8,14 +8,10 @@
"context"
"flag"
"fmt"
- "io/ioutil"
- "os"
- "os/exec"
- "path/filepath"
+ "fuchsia.googlesource.com/tools/botanist/target"
"fuchsia.googlesource.com/tools/build"
"fuchsia.googlesource.com/tools/logger"
- "fuchsia.googlesource.com/tools/qemu"
"fuchsia.googlesource.com/tools/secrets"
"github.com/google/subcommands"
)
@@ -83,58 +79,27 @@
return err
}
- qemuCPU, ok := map[string]string{
- "x64": "x86_64",
- "arm64": "aarch64",
- }[cmd.targetArch]
- if !ok {
- return fmt.Errorf("cpu %q not recognized", cmd.targetArch)
- }
- qemuBinPath := filepath.Join(cmd.qemuBinDir, fmt.Sprintf("%s-%s", qemuBinPrefix, qemuCPU))
-
- cfg := qemu.Config{
- QEMUBin: qemuBinPath,
- CPU: cmd.targetArch,
- KVM: cmd.enableKVM,
- MinFSImage: cmd.minFSImage,
- PCIAddr: cmd.minFSBlkDevPCIAddr,
- InternetAccess: cmd.enableNetworking,
+ // TODO: pass this directly from a file.
+ config := target.QEMUConfig{
+ CPU: 4,
+ Memory: 4096,
+ Path: cmd.qemuBinDir,
+ Target: cmd.targetArch,
+ KVM: cmd.enableKVM,
+ Network: cmd.enableNetworking,
+ MinFS: &target.MinFS{
+ Image: cmd.minFSImage,
+ PCIAddress: cmd.minFSBlkDevPCIAddr,
+ },
}
- // The system will halt on a kernel panic instead of rebooting
- cmdlineArgs = append(cmdlineArgs, "kernel.halt-on-panic=true")
- // Print a message if `dm poweroff` times out.
- cmdlineArgs = append(cmdlineArgs, "devmgr.suspend-timeout-debug=true")
- // Do not print colors.
- cmdlineArgs = append(cmdlineArgs, "TERM=dumb")
- if cmd.targetArch == "x64" {
- // Necessary to redirect to stdout.
- cmdlineArgs = append(cmdlineArgs, "kernel.serial=legacy")
- }
+ t := target.NewQEMUTarget(config)
- invocation, err := qemu.CreateInvocation(cfg, imgs, cmdlineArgs)
- if err != nil {
+ if err := t.Start(ctx, imgs, cmdlineArgs); err != nil {
return err
}
- // The QEMU command needs to be invoked within an empty directory, as QEMU will attempt
- // to pick up files from its working directory, one notable culprit being multiboot.bin.
- // This can result in strange behavior.
- qemuWorkingDir, err := ioutil.TempDir("", "qemu-working-dir")
- if err != nil {
- return err
- }
- defer os.RemoveAll(qemuWorkingDir)
-
- qemuCmd := exec.Cmd{
- Path: invocation[0],
- Args: invocation,
- Dir: qemuWorkingDir,
- Stdout: os.Stdout,
- Stderr: os.Stderr,
- }
- logger.Debugf(ctx, "QEMU invocation:\n%s", invocation)
- return qemu.CheckExitCode(qemuCmd.Run())
+ return t.Wait(ctx)
}
func (cmd *QEMUCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
diff --git a/qemu/config.go b/qemu/config.go
index a9583d8..495e770 100644
--- a/qemu/config.go
+++ b/qemu/config.go
@@ -5,157 +5,190 @@
package qemu
import (
- "errors"
"fmt"
- "io"
- "io/ioutil"
"os"
"path/filepath"
"strings"
-
- "fuchsia.googlesource.com/tools/build"
)
+const (
+ DefaultNetwork = "10.0.2.0/24"
+ DefaultDHCPStart = "10.0.2.15"
+ DefaultGateway = "10.0.2.2"
+ DefaultDNS = "10.0.2.3"
+)
+
+const (
+ TargetAArch64 = "aarch64"
+ TargetX86_64 = "x86_64"
+)
+
+type Drive struct {
+ // ID is the block device identifier.
+ ID string
+
+ // File is the disk image file.
+ File string
+
+ // Addr is the PCI address of the block device.
+ Addr string
+}
+
+type Forward struct {
+ // HostPort is the port on the host.
+ HostPort int
+
+ // GuestPort is the port on the guest.
+ GuestPort int
+}
+
+type Netdev struct {
+ // ID is the network device identifier.
+ ID string
+
+ // Network is the network block.
+ Network string
+
+ // DHCPStart is the address at which the DHCP allocation starts.
+ DHCPStart string
+
+ // DNS is the address of the builtin DNS server.
+ DNS string
+
+ // Host is the host IP address.
+ Host string
+
+ // Forwards are the host forwardings.
+ Forwards []Forward
+
+ // MAC is the network device MAC address.
+ MAC string
+}
+
// Config gives a high-level configuration for QEMU on Fuchsia.
type Config struct {
// QEMUBin is a path to the QEMU binary.
- QEMUBin string
+ Binary string
- // CPU is the emulated CPU (e.g., "x64" or "arm64").
- CPU string
+ // Target is the QEMU target (e.g., "x86_64" or "aarch64").
+ Target string
+
+ // CPU is the number of CPUs.
+ CPU int
+
+ // Memory is the amount of RAM.
+ Memory int
// KVM gives whether to enable KVM.
KVM bool
- // MinFSImage is the path to a minfs image. If unset, none will be attached.
- MinFSImage string
+ // Kernel is the path to the kernel image.
+ Kernel string
- // PCIAddr is the PCI address under which a minfs image will be mounted as a block device.
- PCIAddr string
+ // Initrd is the path to the initrd image.
+ Initrd string
- // InternetAccess gives whether to enable internet access.
- InternetAccess bool
+ // Drives are drives to mount inside the QEMU instance.
+ Drives []Drive
+
+ // Networks are networks to set up inside the QEMU instance.
+ Networks []Netdev
}
// CreateInvocation creates a QEMU invocation given a particular configuration, a list of
// images, and any specified command-line arguments.
-func CreateInvocation(cfg Config, imgs build.Images, cmdlineArgs []string) ([]string, error) {
- if _, err := os.Stat(cfg.QEMUBin); err != nil {
+func CreateInvocation(cfg Config, cmdlineArgs []string) ([]string, error) {
+ if _, err := os.Stat(cfg.Binary); err != nil {
return nil, fmt.Errorf("QEMU binary not found: %v", err)
}
- absQEMUBinPath, err := filepath.Abs(cfg.QEMUBin)
+ absBinaryPath, err := filepath.Abs(cfg.Binary)
if err != nil {
return nil, err
}
- invocation := []string{absQEMUBinPath}
- addArgs := func(args ...string) {
- invocation = append(invocation, args...)
- }
+ invocation := []string{absBinaryPath}
- if cfg.CPU == "arm64" {
+ switch cfg.Target {
+ case TargetAArch64:
if cfg.KVM {
- addArgs("-machine", "virt,gic_version=host")
- addArgs("-cpu", "host")
- addArgs("-enable-kvm")
+ invocation = append(invocation, "-machine", "virt,gic_version=host")
+ invocation = append(invocation, "-cpu", "host")
+ invocation = append(invocation, "-enable-kvm")
} else {
- addArgs("-machine", "virt,gic_version=3")
- addArgs("-machine", "virtualization=true")
- addArgs("-cpu", "cortex-a53")
+ invocation = append(invocation, "-machine", "virt,gic_version=3")
+ invocation = append(invocation, "-machine", "virtualization=true")
+ invocation = append(invocation, "-cpu", "cortex-a53")
}
- } else if cfg.CPU == "x64" {
- addArgs("-machine", "q35")
- // Necessary for userboot.shutdown to trigger properly, since it writes to
- // 0xf4 to debug-exit in QEMU.
- addArgs("-device", "isa-debug-exit,iobase=0xf4,iosize=0x04")
-
+ case TargetX86_64:
+ invocation = append(invocation, "-machine", "q35")
+ // TODO: this is Fuchsia specific, factor it out as another device struct.
+ // Necessary for userboot.shutdown to trigger properly, since it writes
+ // to 0xf4 to debug-exit in QEMU.
+ invocation = append(invocation, "-device", "isa-debug-exit,iobase=0xf4,iosize=0x04")
if cfg.KVM {
- addArgs("-cpu", "host")
- addArgs("-enable-kvm")
+ invocation = append(invocation, "-cpu", "host")
+ invocation = append(invocation, "-enable-kvm")
} else {
- addArgs("-cpu", "Haswell,+smap,-check,-fsgsbase")
+ invocation = append(invocation, "-cpu", "Haswell,+smap,-check,-fsgsbase")
}
- } else {
- return nil, fmt.Errorf("cpu %q not recognized", cfg.CPU)
+ default:
+ return nil, fmt.Errorf("cpu %q not recognized", cfg.Target)
}
- addArgs("-m", "4096")
- addArgs("-smp", "4")
- addArgs("-nographic")
- addArgs("-serial", "stdio")
- addArgs("-monitor", "none")
+ invocation = append(invocation, "-m", fmt.Sprintf("%d", cfg.Memory))
+ invocation = append(invocation, "-smp", fmt.Sprintf("%d", cfg.CPU))
+ invocation = append(invocation, "-nographic")
+ invocation = append(invocation, "-serial", "stdio")
+ invocation = append(invocation, "-monitor", "none")
- if !cfg.InternetAccess {
- addArgs("-net", "none")
- }
+ invocation = append(invocation, "-kernel", cfg.Kernel)
+ invocation = append(invocation, "-initrd", cfg.Initrd)
- if cfg.MinFSImage != "" {
- if cfg.PCIAddr == "" {
- return nil, errors.New("PCI address must be set if a MinFS image is provided")
+ // TODO: maybe we should introduce Device interface with three different
+ // implementations: Drive, Netdev and ISADebugExit to cleanup the code
+ // below a bit.
+
+ for _, d := range cfg.Drives {
+ var drive strings.Builder
+ fmt.Fprintf(&drive, "id=%s,file=%s,format=raw,if=none", d.ID, d.File)
+ invocation = append(invocation, "-drive", drive.String())
+
+ var device strings.Builder
+ fmt.Fprintf(&device, "virtio-blk-pci,drive=%s", d.ID)
+ if d.Addr != "" {
+ fmt.Fprintf(&device, ",addr=%s", d.Addr)
}
- absMinFSImage, err := filepath.Abs(cfg.MinFSImage)
- if err != nil {
- return nil, err
+ invocation = append(invocation, "-device", device.String())
+ }
+
+ for _, n := range cfg.Networks {
+ var netdev strings.Builder
+ fmt.Fprintf(&netdev, "user,id=%s", n.ID)
+ if n.Network != "" {
+ fmt.Fprintf(&netdev, ",net=%s", n.Network)
}
- // Swarming hard-links Isolate downloads with a cache and the very same cached minfs
- // image will be used across multiple tasks. To ensure that it remains blank, we
- // must break its link.
- if err := overwriteFileWithCopy(absMinFSImage); err != nil {
- return nil, err
+ if n.DHCPStart != "" {
+ fmt.Fprintf(&netdev, ",dhcpstart=%s", n.DHCPStart)
}
- addArgs("-drive", fmt.Sprintf("file=%s,format=raw,if=none,id=testdisk", absMinFSImage))
- addArgs("-device", fmt.Sprintf("virtio-blk-pci,drive=testdisk,addr=%s", cfg.PCIAddr))
+ if n.DNS != "" {
+ fmt.Fprintf(&netdev, ",dns=%s", n.DNS)
+ }
+ if n.Host != "" {
+ fmt.Fprintf(&netdev, ",host=%s", n.Host)
+ }
+ for _, f := range n.Forwards {
+ fmt.Fprintf(&netdev, ",hostfwd=tcp::%d-:%d", f.HostPort, f.GuestPort)
+ }
+ invocation = append(invocation, "-netdev", netdev.String())
+
+ var device strings.Builder
+ fmt.Fprintf(&device, "virtio-net-pci,netdev=%s", n.ID)
+ if n.MAC != "" {
+ fmt.Fprintf(&device, ",mac=%s", n.MAC)
+ }
+ invocation = append(invocation, "-device", device.String())
}
- qemuKernel := imgs.Get("qemu-kernel")
- if qemuKernel == nil {
- return nil, fmt.Errorf("could not find qemu-kernel")
- }
- zirconA := imgs.Get("zircon-a")
- if zirconA == nil {
- return nil, fmt.Errorf("could not find zircon-a")
- }
- addArgs("-kernel", qemuKernel.Path)
- addArgs("-initrd", zirconA.Path)
-
- if storageFull := imgs.Get("storage-full"); storageFull != nil {
- addArgs("-drive", fmt.Sprintf("file=%s,format=raw,if=none,id=maindisk", storageFull.Path))
- addArgs("-device", "virtio-blk-pci,drive=maindisk")
- }
-
- addArgs("-append", strings.Join(cmdlineArgs, " "))
+ invocation = append(invocation, "-append", strings.Join(cmdlineArgs, " "))
return invocation, nil
}
-
-func overwriteFileWithCopy(path string) error {
- copy, err := ioutil.TempFile("", "botanist")
- if err != nil {
- return err
- }
- if err = copyFile(path, copy.Name()); err != nil {
- return err
- }
- if err = os.Remove(path); err != nil {
- return err
- }
- return os.Rename(copy.Name(), path)
-}
-
-func copyFile(src, dest string) error {
- in, err := os.Open(src)
- if err != nil {
- return err
- }
- defer in.Close()
- info, err := in.Stat()
- if err != nil {
- return err
- }
- out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, info.Mode().Perm())
- if err != nil {
- return err
- }
- defer out.Close()
- _, err = io.Copy(out, in)
- return err
-}
