[tarutil] Create pkg for tarring data
- Adds tests for Tar archive code.
- Replaces:
- botanist.ArchiveBuffer -> tarutil.TarBuffer
- botanist.ArchiveReader -> tarutil.TarReader
- botanist.ArchiveDirectory -> tarutil.TarDirectory
- Ran goimports to fix imports which also formatted some files.
(everyone please remember to run gofmt before submitting)
These things aren't botanist specific. Using the name "TarXXX" makes
it possible to add different archive formats.
Change-Id: I138a2636bcd475c1091a609d143c9556ea15e803
diff --git a/botanist/fileutil.go b/botanist/fileutil.go
index 7e4e06f..f492e43 100644
--- a/botanist/fileutil.go
+++ b/botanist/fileutil.go
@@ -7,66 +7,11 @@
import (
"archive/tar"
"fmt"
- "io"
- "io/ioutil"
"net"
- "os"
- "path/filepath"
"fuchsia.googlesource.com/tools/tftp"
)
-// ArchiveDirectory archives the given directory.
-func ArchiveDirectory(tw *tar.Writer, dir string) error {
- return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- if info.IsDir() {
- return nil
- }
-
- hdr, err := tar.FileInfoHeader(info, path)
- if err != nil {
- return err
- }
- hdr.Name = path[len(dir)+1:]
- if err := tw.WriteHeader(hdr); err != nil {
- return err
- }
- fi, err := os.Open(path)
- if err != nil {
- return err
- }
- _, err = io.Copy(tw, fi)
- return err
- })
-}
-
-// ArchiveBuffer writes the given bytes to a given path within an archive.
-func ArchiveBuffer(tw *tar.Writer, buf []byte, path string) error {
- hdr := &tar.Header{
- Name: path,
- Size: int64(len(buf)),
- Mode: 0666,
- }
- if err := tw.WriteHeader(hdr); err != nil {
- return err
- }
- _, err := tw.Write(buf)
- return err
-}
-
-// ArchiveReader writes data from the given Reader to the given tar.Writer.
-func ArchiveReader(tw *tar.Writer, r io.Reader, path string) error {
- bytes, err := ioutil.ReadAll(r)
- if err != nil {
- return err
- }
-
- return ArchiveBuffer(tw, bytes, path)
-}
-
// FetchAndArchiveFile fetches a remote file via TFTP from a given node, and
// writes it an archive.
func FetchAndArchiveFile(t *tftp.Client, addr *net.UDPAddr, tw *tar.Writer, path, name string) error {
diff --git a/botanist/target/qemu.go b/botanist/target/qemu.go
index 050c10f..7b8f816 100644
--- a/botanist/target/qemu.go
+++ b/botanist/target/qemu.go
@@ -28,11 +28,10 @@
defaultInterfaceName = "qemu"
// DefaultMACAddr is the default MAC address given to a QEMU target.
- defaultMACAddr = "52:54:00:63:5e:7a"
+ defaultMACAddr = "52:54:00:63:5e:7a"
// DefaultNodename is the default nodename given to an target with the default QEMU MAC address.
- defaultNodename = "step-atom-yard-juicy"
-
+ defaultNodename = "step-atom-yard-juicy"
)
// qemuTargetMapping maps the Fuchsia target name to the name recognized by QEMU.
@@ -98,7 +97,7 @@
// IPv4Addr returns a nil address, as DHCP is not currently configured.
func (t *QEMUTarget) IPv4Addr() (net.IP, error) {
- return nil, nil
+ return nil, nil
}
// SSHKey returns the private SSH key path associated with the authorized key to be pavet.
diff --git a/cmd/artifacts/storetestoutputs.go b/cmd/artifacts/storetestoutputs.go
index 83951c4..b9aec9f 100644
--- a/cmd/artifacts/storetestoutputs.go
+++ b/cmd/artifacts/storetestoutputs.go
@@ -44,10 +44,10 @@
// StoreTestOutputsCommand performs a batch upload of test outputs to Cloud Storage.
type StoreTestOutputsCommand struct {
authFlags authcli.Flags
- bucket string
- build string
- testEnv string
- workers sync.WaitGroup
+ bucket string
+ build string
+ testEnv string
+ workers sync.WaitGroup
}
func (*StoreTestOutputsCommand) Name() string {
diff --git a/cmd/botanist/zedboot.go b/cmd/botanist/zedboot.go
index 052e32f..7999d9d 100644
--- a/cmd/botanist/zedboot.go
+++ b/cmd/botanist/zedboot.go
@@ -17,7 +17,7 @@
"strings"
"time"
- "fuchsia.googlesource.com/tools/botanist"
+ "fuchsia.googlesource.com/tools/tarutil"
"fuchsia.googlesource.com/tools/botanist/target"
"fuchsia.googlesource.com/tools/build"
"fuchsia.googlesource.com/tools/command"
@@ -135,17 +135,17 @@
defer tw.Close()
// Write summary to archive
- if err = botanist.ArchiveBuffer(tw, summary, cmd.summaryFilename); err != nil {
+ if err = tarutil.TarBuffer(tw, summary, cmd.summaryFilename); err != nil {
return err
}
// Write combined stdout & stderr output to archive
- if err = botanist.ArchiveBuffer(tw, cmdOutput, runtests.TestOutputFilename); err != nil {
+ if err = tarutil.TarBuffer(tw, cmdOutput, runtests.TestOutputFilename); err != nil {
return err
}
// Write all output files from the host cmd to the archive.
- return botanist.ArchiveDirectory(tw, outputDir)
+ return tarutil.TarDirectory(tw, outputDir)
}
// Executes host command and creates result tar from command output
diff --git a/cmd/buildidtool/main.go b/cmd/buildidtool/main.go
index fa246f1..0aa14e3 100644
--- a/cmd/buildidtool/main.go
+++ b/cmd/buildidtool/main.go
@@ -149,11 +149,11 @@
buildID := []rune(hex.EncodeToString(buildIDs[0]))
buildIDPath := filepath.Join(buildIDDir, string(buildID[:2]), string(buildID[2:])) + extension
// Now perform the operations of the tool. The order in which these operations occur
- // ensures that, from the perspective of the build system, all these operations occur
- // atomically. This order is "valid" because unless the tool runs to the end
- // then ninja will rerun the step and when the step is rerun once finished the end
- // state will be valid. The order of the first 3 steps doesn't matter much but the
- // stamp file must be emitted last.
+ // ensures that, from the perspective of the build system, all these operations occur
+ // atomically. This order is "valid" because unless the tool runs to the end
+ // then ninja will rerun the step and when the step is rerun once finished the end
+ // state will be valid. The order of the first 3 steps doesn't matter much but the
+ // stamp file must be emitted last.
if err = atomicLink(file, buildIDPath); err != nil {
l.Fatalf("atomically linking %s to %s: %v", file, buildIDPath, err)
}
diff --git a/cmd/testrunner/outputs/tar.go b/cmd/testrunner/outputs/tar.go
index b635853..194168e 100644
--- a/cmd/testrunner/outputs/tar.go
+++ b/cmd/testrunner/outputs/tar.go
@@ -10,7 +10,7 @@
"io"
"path"
- "fuchsia.googlesource.com/tools/botanist"
+ "fuchsia.googlesource.com/tools/tarutil"
"fuchsia.googlesource.com/tools/runtests"
"fuchsia.googlesource.com/tools/testrunner"
)
@@ -31,12 +31,12 @@
pathInArchive := path.Join(result.Name, runtests.TestOutputFilename)
stdout := bytes.NewReader(result.Stdout)
stderr := bytes.NewReader(result.Stderr)
- botanist.ArchiveReader(o.w, io.MultiReader(stdout, stderr), pathInArchive)
+ tarutil.TarReader(o.w, io.MultiReader(stdout, stderr), pathInArchive)
}
// TarFile adds a file to the underlying archive.
func (o *TarOutput) TarFile(bytes []byte, filename string) error {
- return botanist.ArchiveBuffer(o.w, bytes, filename)
+ return tarutil.TarBuffer(o.w, bytes, filename)
}
// Close flushes all data to the archive.
diff --git a/resultstore/mocks/proto_mocks.go b/resultstore/mocks/proto_mocks.go
index d17e5d1..860d3ef 100644
--- a/resultstore/mocks/proto_mocks.go
+++ b/resultstore/mocks/proto_mocks.go
@@ -10,10 +10,11 @@
import (
context "context"
+ reflect "reflect"
+
gomock "github.com/golang/mock/gomock"
v2 "google.golang.org/genproto/googleapis/devtools/resultstore/v2"
grpc "google.golang.org/grpc"
- reflect "reflect"
)
// MockResultStoreUploadClient is a mock of ResultStoreUploadClient interface
diff --git a/runtests/poll.go b/runtests/poll.go
index 08b7b5f..f251374 100644
--- a/runtests/poll.go
+++ b/runtests/poll.go
@@ -19,6 +19,7 @@
"fuchsia.googlesource.com/tools/botanist"
"fuchsia.googlesource.com/tools/logger"
"fuchsia.googlesource.com/tools/retry"
+ "fuchsia.googlesource.com/tools/tarutil"
"fuchsia.googlesource.com/tools/tftp"
)
@@ -62,7 +63,7 @@
tw := tar.NewWriter(outFile)
defer tw.Close()
- if err = botanist.ArchiveBuffer(tw, buffer.Bytes(), summaryFilename); err != nil {
+ if err = tarutil.TarBuffer(tw, buffer.Bytes(), summaryFilename); err != nil {
return err
}
diff --git a/secrets/server_test.go b/secrets/server_test.go
index 21d6e20..e761373 100644
--- a/secrets/server_test.go
+++ b/secrets/server_test.go
@@ -5,11 +5,12 @@
import (
"context"
- "go.chromium.org/luci/lucictx"
"net/http"
"net/http/httptest"
"reflect"
"testing"
+
+ "go.chromium.org/luci/lucictx"
)
func TestGettingSecrets(t *testing.T) {
diff --git a/tarutil/tar.go b/tarutil/tar.go
new file mode 100644
index 0000000..842f8bd
--- /dev/null
+++ b/tarutil/tar.go
@@ -0,0 +1,63 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package tarutil
+
+import (
+ "archive/tar"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+)
+
+// TarDirectory archives the given directory.
+func TarDirectory(tw *tar.Writer, dir string) error {
+ return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+
+ hdr, err := tar.FileInfoHeader(info, path)
+ if err != nil {
+ return err
+ }
+ hdr.Name = path[len(dir)+1:]
+ if err := tw.WriteHeader(hdr); err != nil {
+ return err
+ }
+ fi, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(tw, fi)
+ return err
+ })
+}
+
+// TarBuffer writes the given bytes to a given path within an archive.
+func TarBuffer(tw *tar.Writer, buf []byte, path string) error {
+ hdr := &tar.Header{
+ Name: path,
+ Size: int64(len(buf)),
+ Mode: 0666,
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ return err
+ }
+ _, err := tw.Write(buf)
+ return err
+}
+
+// TarReader writes data from the given Reader to the given tar.Writer.
+func TarReader(tw *tar.Writer, r io.Reader, path string) error {
+ bytes, err := ioutil.ReadAll(r)
+ if err != nil {
+ return err
+ }
+ return TarBuffer(tw, bytes, path)
+}
diff --git a/tarutil/tar_test.go b/tarutil/tar_test.go
new file mode 100644
index 0000000..eb66d30
--- /dev/null
+++ b/tarutil/tar_test.go
@@ -0,0 +1,101 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package tarutil_test
+
+import (
+ "archive/tar"
+ "bytes"
+ "fmt"
+ "io"
+ "reflect"
+ "testing"
+
+ "fuchsia.googlesource.com/tools/tarutil"
+)
+
+func TestTarBuffer(t *testing.T) {
+ type entry struct {
+ name, data string
+ }
+
+ tests := []struct {
+ // A name for this test case.
+ name string
+
+ // The input tarball.
+ input []entry
+
+ // The expected contents of the archive written by WriteTo.
+ output map[string]string
+ }{
+ {
+ name: "should handle an empty buffer",
+ input: []entry{{"", ""}},
+ output: map[string]string{"": ""},
+ },
+ {
+ name: "should handle a non-empty buffer",
+ input: []entry{{"a", string("a data")}},
+ output: map[string]string{
+ "a": string("a data"),
+ },
+ },
+ {
+ name: "should handle multiple non-empty buffers",
+ input: []entry{
+ {"a", string("a data")},
+ {"b", string("b data")},
+ },
+ output: map[string]string{
+ "a": string("a data"),
+ "b": string("b data"),
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var buf bytes.Buffer
+ tw := tar.NewWriter(&buf)
+ for _, ent := range tt.input {
+ tarutil.TarBuffer(tw, []byte(ent.data), ent.name)
+ }
+ actual, err := readTar(&buf)
+ if err != nil {
+ t.Errorf("failed to read tar archive: %v", err)
+ return
+ }
+ expected := tt.output
+ if !reflect.DeepEqual(actual, expected) {
+ t.Errorf("got:\n\n%v\n\nwanted:\n\n%v\n\n", actual, expected)
+ }
+ })
+ }
+
+}
+
+// Helper function to read data from a gzipped tar archive. The output maps each header's
+// name within the archive to its data.
+func readTar(r io.Reader) (map[string]string, error) {
+ tr := tar.NewReader(r)
+ output := make(map[string]string)
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ break // End of archive.
+ }
+ if err != nil {
+ return nil, fmt.Errorf("reading tarball failed, %v", err)
+
+ }
+ data := make([]byte, hdr.Size)
+ if _, err := tr.Read(data); err != nil && err != io.EOF {
+ return nil, fmt.Errorf("reading tarball data failed, %v", err)
+ }
+ output[hdr.Name] = string(data)
+ }
+
+ return output, nil
+}
