[bootserver] Download images to file before transferring.
This will help with the attached bug because we can't properly get the
uncompressed size of compressed images from GCS. If we write it to disc,
we can pass the file and it's size to the tftp transfer.
Bug: 43935
Change-Id: I623ee623655dec706d0c8a4968a37166e42f4664
diff --git a/tools/bootserver/BUILD.gn b/tools/bootserver/BUILD.gn
index 2b42d94..655859f9 100644
--- a/tools/bootserver/BUILD.gn
+++ b/tools/bootserver/BUILD.gn
@@ -12,6 +12,7 @@
"//third_party/golibs:cloud.google.com/go/storage",
"//third_party/golibs:golang.org/x/crypto",
"//tools/build",
+ "//tools/lib/iomisc",
"//tools/lib/logger",
"//tools/lib/retry",
"//tools/net/netboot",
diff --git a/tools/bootserver/boot.go b/tools/bootserver/boot.go
index a37a64a..fbb79b1 100644
--- a/tools/bootserver/boot.go
+++ b/tools/bootserver/boot.go
@@ -11,9 +11,13 @@
"fmt"
"io"
"log"
+ "os"
+ "path/filepath"
"sort"
+ "sync"
"time"
+ "go.fuchsia.dev/fuchsia/tools/lib/iomisc"
"go.fuchsia.dev/fuchsia/tools/lib/retry"
"go.fuchsia.dev/fuchsia/tools/net/netboot"
"go.fuchsia.dev/fuchsia/tools/net/tftp"
@@ -71,6 +75,88 @@
kernelNetsvcName: 13,
}
+func downloadImages(imgs []Image) ([]Image, func() error, error) {
+ var newImgs []Image
+ // Copy each in a goroutine for efficiency's sake.
+ errs := make(chan error, len(imgs))
+ var mux sync.Mutex
+ var wg sync.WaitGroup
+ for _, img := range imgs {
+ wg.Add(1)
+ go func(img Image) {
+ defer wg.Done()
+ if img.Reader != nil {
+ f, err := downloadAndOpenImage(img.Name, img)
+ if err != nil {
+ errs <- err
+ return
+ }
+ fi, err := f.Stat()
+ if err != nil {
+ f.Close()
+ errs <- err
+ return
+ }
+ mux.Lock()
+ newImgs = append(newImgs, Image{
+ Name: img.Name,
+ Reader: f,
+ Size: fi.Size(),
+ Args: img.Args,
+ })
+ mux.Unlock()
+ }
+ }(img)
+ }
+ wg.Wait()
+
+ closeFunc := func() error { return closeImages(newImgs) }
+ select {
+ case err := <-errs:
+ closeImages(newImgs)
+ return nil, closeFunc, err
+ default:
+ return newImgs, closeFunc, nil
+ }
+}
+
+func downloadAndOpenImage(dest string, img Image) (*os.File, error) {
+ f, ok := img.Reader.(*os.File)
+ if ok {
+ return f, nil
+ }
+
+ // If the file already exists at dest, just open and return the file instead of downloading again.
+ // This will avoid duplicate downloads from catalyst (which calls the bootserver tool) and botanist.
+ if _, err := os.Stat(dest); !os.IsNotExist(err) {
+ return os.Open(dest)
+ }
+
+ f, err := os.Create(dest)
+ if err != nil {
+ return nil, err
+ }
+
+ // Log progress to avoid hitting I/O timeout in case of slow transfers.
+ ticker := time.NewTicker(30 * time.Second)
+ defer ticker.Stop()
+ go func() {
+ for range ticker.C {
+ log.Printf("transferring %s...\n", filepath.Base(dest))
+ }
+ }()
+
+ if _, err := io.Copy(f, iomisc.ReaderAtToReader(img.Reader)); err != nil {
+ f.Close()
+ return nil, fmt.Errorf("failed to copy image %q to %q: %v", img.Name, dest, err)
+ }
+ if err := f.Sync(); err != nil {
+ f.Close()
+ return nil, err
+ }
+ return f, nil
+}
+
// Boot prepares and boots a device at the given IP address.
func Boot(ctx context.Context, t tftp.Client, imgs []Image, cmdlineArgs []string, signers []ssh.Signer) error {
var files []*netsvcFile
@@ -87,6 +173,14 @@
files = append(files, cmdlineFile)
}
+ // This is needed because imgs from GCS are compressed and we cannot get the correct size of the uncompressed images, so we have to download them first.
+ // TODO(ihuh): We should enable this step as a command line option.
+ imgs, closeFunc, err := downloadImages(imgs)
+ if err != nil {
+ return err
+ }
+ defer closeFunc()
+
for _, img := range imgs {
for _, arg := range img.Args {
name, ok := bootserverArgToName[arg]
diff --git a/tools/bootserver/boot_test.go b/tools/bootserver/boot_test.go
new file mode 100644
index 0000000..86c83ab
--- /dev/null
+++ b/tools/bootserver/boot_test.go
@@ -0,0 +1,39 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package bootserver
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestDownloadImage(t *testing.T) {
+ tmpDir, err := ioutil.TempDir("", "test-data")
+ if err != nil {
+ t.Fatalf("failed to create temp dir: %v", err)
+ }
+ defer os.RemoveAll(tmpDir)
+ expectedData := "content for test image to download"
+ reader := bytes.NewReader([]byte(expectedData))
+ imgPath := filepath.Join(tmpDir, "image")
+ f, err := downloadAndOpenImage(imgPath, Image{
+ Name: "image",
+ Reader: reader,
+ })
+ if err != nil {
+ t.Fatalf("failed to download image: %v", err)
+ }
+ f.Close()
+ content, err := ioutil.ReadFile(imgPath)
+ if err != nil {
+ t.Fatalf("failed to read file: %v", err)
+ }
+ if string(content) != expectedData {
+ t.Fatalf("unexpected content: expected: %s, actual: %s", expectedData, content)
+ }
+}
