[testrunner] Add testrunner binary
IN-824 #comment
Change-Id: I04223efa0dcbd32daaaed899fae2d6c11a6dd9ac
diff --git a/cmd/testrunner/main.go b/cmd/testrunner/main.go
new file mode 100644
index 0000000..ded31f8
--- /dev/null
+++ b/cmd/testrunner/main.go
@@ -0,0 +1,216 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "fuchsia.googlesource.com/tools/botanist"
+ "fuchsia.googlesource.com/tools/runtests"
+ "fuchsia.googlesource.com/tools/testsharder"
+ "golang.org/x/crypto/ssh"
+)
+
+// TODO(IN-824): Produce a tar archive of all output files.
+// TODO(IN-824): Include log_listener output.
+
+const (
+ // Default amount of time to wait before failing to perform any IO action.
+ defaultIOTimeout = 1 * time.Minute
+
+ // The username used to authenticate with the Fuchsia device.
+ sshUser = "fuchsia"
+
+ // The test output directory to create on the Fuchsia device.
+ fuchsiaOutputDir = "/data/infra/testrunner"
+)
+
+// Command-line flags
+var (
+ // Whether to show Usage and exit.
+ help bool
+
+ // The directory where output should be written. This path will contain both
+ // summary.json and the set of output files for each test.
+ outputDir string
+
+ // The path to a file containing properties of the Fuchsia device to use for testing.
+ deviceFilepath string
+)
+
+func usage() {
+ fmt.Println(`
+ testrunner [flags] tests-file
+
+ Executes all tests found in the JSON [tests-file]
+ Required environment variables:
+
+ "NODENAME": The nodename of the attached Fuchsia device to use for testing. This
+ can usually be found using the 'netls' tool.
+ "SSH_KEY": Path to the SSH private key used to connect to the Fuchsia device.
+ `)
+}
+
+func init() {
+ flag.BoolVar(&help, "help", false, "Whether to show Usage and exit.")
+ flag.StringVar(&outputDir, "output", "", "Directory where output should be written")
+ flag.Usage = usage
+}
+
+func main() {
+ flag.Parse()
+
+ if help || flag.NArg() != 1 {
+ flag.Usage()
+ flag.PrintDefaults()
+ return
+ }
+
+ if outputDir == "" {
+ log.Fatal("-output is required")
+ }
+
+ if err := execute(flag.Arg(0)); err != nil {
+ log.Fatal(err)
+ }
+}
+
+func execute(testsFilepath string) error {
+ // Validate inputs.
+ nodename := os.Getenv("NODENAME")
+ if nodename == "" {
+ return errors.New("missing environment variable NODENAME")
+ }
+
+ privateKeyPath := os.Getenv("SSH_KEY")
+ if privateKeyPath == "" {
+ return errors.New("missing environment variable SSH_KEY")
+ }
+
+ // Initialize the connection to the Fuchsia device.
+ sshClient, err := sshIntoNode(nodename, privateKeyPath)
+ if err != nil {
+ return fmt.Errorf("failed to connect to node %q: %v", nodename, err)
+ }
+
+ fuchsiaTester := &FuchsiaTester{
+ remoteOutputDir: fuchsiaOutputDir,
+ delegate: &SSHTester{
+ client: sshClient,
+ },
+ }
+
+ // Parse test input.
+ bytes, err := ioutil.ReadFile(testsFilepath)
+ if err != nil {
+ return fmt.Errorf("failed to read %s: %v", testsFilepath, err)
+ }
+
+ var tests []testsharder.Test
+ if err := json.Unmarshal(bytes, &tests); err != nil {
+ return fmt.Errorf("failed to unmarshal %s: %v", testsFilepath, err)
+ }
+
+ // Execute all tests.
+ summary, err := runTests(tests, fuchsiaTester.Test, RunTestInSubprocess, outputDir)
+ if err != nil {
+ return err
+ }
+
+ summaryFile, err := os.Create(path.Join(outputDir, "summary.json"))
+ if err != nil {
+ return err
+ }
+
+ // Log summary to `outputDir`.
+ encoder := json.NewEncoder(summaryFile)
+ return encoder.Encode(summary)
+}
+
+func sshIntoNode(nodename, privateKeyPath string) (*ssh.Client, error) {
+ privateKey, err := ioutil.ReadFile(privateKeyPath)
+ if err != nil {
+ return nil, err
+ }
+
+ signer, err := ssh.ParsePrivateKey(privateKey)
+ if err != nil {
+ return nil, err
+ }
+
+ config := &ssh.ClientConfig{
+ User: sshUser,
+ Auth: []ssh.AuthMethod{
+ ssh.PublicKeys(signer),
+ },
+ Timeout: defaultIOTimeout,
+ HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+ }
+
+ return botanist.SSHIntoNode(context.Background(), nodename, config)
+}
+
+func runTests(tests []testsharder.Test, fuchsia Tester, local Tester, outputDir string) (*runtests.TestSummary, error) {
+ // Execute all tests.
+ summary := new(runtests.TestSummary)
+ for _, test := range tests {
+ var tester Tester
+ switch strings.ToLower(test.OS) {
+ case "fuchsia":
+ tester = fuchsia
+ case "linux", "mac":
+ tester = local
+ default:
+ log.Printf("cannot run '%s' on unknown OS '%s'", test.Name, test.OS)
+ continue
+ }
+
+ details, err := runTest(context.Background(), test, tester, outputDir)
+ if err != nil {
+ log.Println(err)
+ }
+
+ if details != nil {
+ summary.Tests = append(summary.Tests, *details)
+ }
+ }
+
+ return summary, nil
+}
+
+func runTest(ctx context.Context, test testsharder.Test, tester Tester, outputDir string) (*runtests.TestDetails, error) {
+ // Create a file for test output.
+ output, err := os.Create(path.Join(outputDir, test.Name, runtests.TestOutputFilename))
+ if err != nil {
+ return nil, err
+ }
+ defer output.Close()
+
+ result := runtests.TestSuccess
+ multistdout := io.MultiWriter(output, os.Stdout)
+ multistderr := io.MultiWriter(output, os.Stderr)
+ if err := tester(ctx, test, multistdout, multistderr); err != nil {
+ result = runtests.TestFailure
+ log.Println(err)
+ }
+
+ // Record the test details in the summary.
+ return &runtests.TestDetails{
+ Name: test.Name,
+ OutputFile: output.Name(),
+ Result: result,
+ }, nil
+}
diff --git a/cmd/testrunner/tester.go b/cmd/testrunner/tester.go
new file mode 100644
index 0000000..53296d4
--- /dev/null
+++ b/cmd/testrunner/tester.go
@@ -0,0 +1,61 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+ "context"
+ "io"
+
+ "fuchsia.googlesource.com/tools/testrunner"
+ "fuchsia.googlesource.com/tools/testsharder"
+ "golang.org/x/crypto/ssh"
+)
+
+// Tester is executes a Test.
+type Tester func(context.Context, testsharder.Test, io.Writer, io.Writer) error
+
+// RunTestInSubprocess is a TesterFunc that executes the given test in a local subprocess.
+func RunTestInSubprocess(ctx context.Context, test testsharder.Test, stdout io.Writer, stderr io.Writer) error {
+ var command []string
+ if len(test.Location) > 0 {
+ command = []string{test.Location}
+ } else {
+ command = test.Command
+ }
+
+ runner := new(testrunner.SubprocessRunner)
+ return runner.Run(ctx, command, stdout, stderr)
+}
+
+// SSHTester is executes tests over an SSH connection. It assumes the test.Command
+// contains the command line to execute on the remote machine.
+type SSHTester struct {
+ client *ssh.Client
+}
+
+func (t *SSHTester) Test(ctx context.Context, test testsharder.Test, stdout io.Writer, stderr io.Writer) error {
+ session, err := t.client.NewSession()
+ if err != nil {
+ return err
+ }
+ defer session.Close()
+
+ runner := &testrunner.SSHRunner{Session: session}
+ return runner.Run(ctx, test.Command, stdout, stderr)
+}
+
+// This is a hack. We have to run Fuchsia tests using `runtests` on the remote device
+// because there are many ways to execute Fuchsia tests and runtests already does this
+// correctly. This wrapper around SSHTester is meant to keep SSHTester free of OS-specific
+// behavior. Later we'll delete this and use SSHTester directly.
+type FuchsiaTester struct {
+ remoteOutputDir string
+ delegate *SSHTester
+}
+
+func (t *FuchsiaTester) Test(ctx context.Context, test testsharder.Test, stdout io.Writer, stderr io.Writer) error {
+ test.Command = []string{"runtests", "-t", test.Location, "-o", t.remoteOutputDir + "runtests"}
+ return t.delegate.Test(ctx, test, stdout, stderr)
+}
diff --git a/cmd/testrunner/tester_test.go b/cmd/testrunner/tester_test.go
new file mode 100644
index 0000000..b182a42
--- /dev/null
+++ b/cmd/testrunner/tester_test.go
@@ -0,0 +1,104 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "strings"
+ "testing"
+
+ "fuchsia.googlesource.com/tools/testsharder"
+)
+
+func TestTester(t *testing.T) {
+ tester := RunTestInSubprocess
+ cases := []testCase{
+ {
+ name: "should run a command a local subprocess",
+ tester: tester,
+ test: testsharder.Test{
+ Name: "hello_world_test",
+ // Assumes that we're running on a Unix system.
+ Command: []string{"/bin/echo", "Hello world!"},
+ },
+ expectedOutput: "Hello world!",
+ },
+ }
+
+ runTestCases(t, cases)
+}
+
+// Verifies that SSHTester can execute tests on a remote device. These tests are
+// only meant for local verification.
+func TestSSHTester(t *testing.T) {
+ t.Skip("ssh tests are meant for local testing only")
+
+ nodename := os.Getenv("NODENAME")
+ if nodename == "" {
+ t.Fatalf("missing environment variable '%s'", "NODENAME")
+ }
+
+ privateKeyPath := os.Getenv("SSH_KEY")
+ if privateKeyPath == "" {
+ t.Fatalf("missing environment variable '%s'", privateKeyPath)
+ }
+
+ client, err := sshIntoNode(nodename, privateKeyPath)
+ if err != nil {
+ t.Fatalf("failed to connect to node '%s': %v", nodename, err)
+ }
+
+ tester := &SSHTester{client: client}
+ cases := []testCase{
+ {
+ name: "should run a command over SSH",
+ tester: tester.Test,
+ test: testsharder.Test{
+ Name: "hello_world_test",
+ // Just 'echo' and not '/bin/echo' because this assumes we're running on
+ // Fuchsia.
+ Command: []string{"echo", "Hello world!"},
+ },
+ expectedOutput: "Hello world!",
+ },
+ {
+ name: "should run successive commands over SSH",
+ tester: tester.Test,
+ test: testsharder.Test{
+ Name: "hello_again_test",
+ Command: []string{"echo", "Hello again!"},
+ },
+ expectedOutput: "Hello again!",
+ },
+ }
+
+ runTestCases(t, cases)
+}
+
+type testCase struct {
+ name string
+ test testsharder.Test
+ tester Tester
+ expectedOutput string
+ expectError bool
+}
+
+func runTestCases(t *testing.T, cases []testCase) {
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ output := new(bytes.Buffer)
+ err := tt.tester(context.Background(), tt.test, output, output)
+ if tt.expectError && err == nil {
+ t.Fatalf("%s: got nil, wanted error", tt.name)
+ } else if !tt.expectError && err != nil {
+ t.Fatalf("%s: got err '%v', wanted nil", tt.name, err)
+ } else if tt.expectedOutput != strings.TrimSpace(output.String()) {
+ t.Fatalf("%s: got output: '%s', want: '%s'", tt.name, output.String(), tt.expectedOutput)
+ }
+ })
+ }
+}
diff --git a/go.mod b/go.mod
index 53cca96..48997a7 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,9 @@
github.com/google/uuid v1.1.0
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 // indirect
+ github.com/kr/fs v0.1.0 // indirect
+ github.com/pkg/errors v0.8.1 // indirect
+ github.com/pkg/sftp v1.8.3
go.chromium.org/luci v0.0.0-20181218015242-20acb618582d
go.opencensus.io v0.18.0 // indirect
golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045 // indirect
diff --git a/go.sum b/go.sum
index 2c3692e..49189fa 100644
--- a/go.sum
+++ b/go.sum
@@ -20,8 +20,14 @@
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.8.3 h1:9jSe2SxTM8/3bXZjtqnkgTBW+lA8db0knZJyns7gpBA=
+github.com/pkg/sftp v1.8.3/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
@@ -33,6 +39,7 @@
golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
