[testrunner] Support basic TAP output
This change produces a TAP output stream to stdout.
The testrunner still produces a summary.json file. We should fix that
and expect upstream consumers to parse the TAP output.
IN-824 #comment
Change-Id: I2e3f5bf590e3149b180a9a5199b7abd6b80c630c
diff --git a/cmd/testrunner/main.go b/cmd/testrunner/main.go
index f4d254c..727d3a1 100644
--- a/cmd/testrunner/main.go
+++ b/cmd/testrunner/main.go
@@ -21,6 +21,7 @@
"fuchsia.googlesource.com/tools/botanist"
"fuchsia.googlesource.com/tools/runtests"
+ "fuchsia.googlesource.com/tools/tap"
"fuchsia.googlesource.com/tools/testrunner"
"fuchsia.googlesource.com/tools/testsharder"
"golang.org/x/crypto/ssh"
@@ -52,6 +53,9 @@
deviceFilepath string
)
+// TestRecorder records the details of test run.
+type TestRecorder func(runtests.TestDetails)
+
func usage() {
fmt.Println(`
testrunner [flags] tests-file
@@ -84,18 +88,40 @@
log.Fatal("-output is required")
}
+ // Load tests.
testsPath := flag.Arg(0)
tests, err := testrunner.LoadTests(testsPath)
if err != nil {
log.Fatalf("failed to load tests from %q: %v", testsPath, err)
}
- if err := execute(tests); err != nil {
+ // Prepare outputs.
+ tapp := tap.NewProducer(os.Stdout)
+ tapp.Plan(len(tests))
+ var summary runtests.TestSummary
+ recordDetails := func(details runtests.TestDetails) {
+ tapp.Ok(details.Result == runtests.TestSuccess, details.Name)
+ summary.Tests = append(summary.Tests, details)
+ }
+
+ // Execute.
+ if err := execute(tests, recordDetails); err != nil {
+ log.Fatal(err)
+ }
+
+ // Write Summary.
+ file, err := os.Create(path.Join(outputDir, "summary.json"))
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ encoder := json.NewEncoder(file)
+ if err := encoder.Encode(summary); err != nil {
log.Fatal(err)
}
}
-func execute(tests []testsharder.Test) error {
+func execute(tests []testsharder.Test, recorder TestRecorder) error {
// Validate inputs.
nodename := os.Getenv("NODENAME")
if nodename == "" {
@@ -135,38 +161,25 @@
return fmt.Errorf("could not determine the runtime system for following tests %v", unknownTests)
}
- var summary runtests.TestSummary
-
// Execute UNIX tests locally, assuming we're running in a UNIX environment.
var localTests []testsharder.Test
localTests = append(localTests, groups["linux"]...)
localTests = append(localTests, groups["mac"]...)
if len(localTests) > 0 {
- details, err := runTests(localTests, RunTestInSubprocess, outputDir)
- if err != nil {
+ if err := runTests(localTests, RunTestInSubprocess, outputDir, recorder); err != nil {
return err
}
- summary.Tests = append(summary.Tests, details...)
}
// Execute Fuchsia tests.
if fuchsiaTests, ok := groups["fuchsia"]; ok {
// TODO(IN-824): Record log_listener output.
- details, err := runTests(fuchsiaTests, fuchsiaTester.Test, outputDir)
- if err != nil {
+ if err := runTests(fuchsiaTests, fuchsiaTester.Test, outputDir, recorder); err != nil {
return err
}
- summary.Tests = append(summary.Tests, details...)
}
- summaryFile, err := os.Create(path.Join(outputDir, "summary.json"))
- if err != nil {
- return err
- }
-
- // Log summary to `outputDir`.
- encoder := json.NewEncoder(summaryFile)
- return encoder.Encode(summary)
+ return nil
}
// groupTests splits a list of tests into named subgroups according to the names returned
@@ -211,8 +224,7 @@
return botanist.SSHIntoNode(context.Background(), nodename, config)
}
-func runTests(tests []testsharder.Test, tester Tester, outputDir string) ([]runtests.TestDetails, error) {
- var output []runtests.TestDetails
+func runTests(tests []testsharder.Test, tester Tester, outputDir string, record TestRecorder) error {
for _, test := range tests {
details, err := runTest(context.Background(), test, tester, outputDir)
if err != nil {
@@ -220,11 +232,11 @@
}
if details != nil {
- output = append(output, *details)
+ record(*details)
}
}
- return output, nil
+ return nil
}
func runTest(ctx context.Context, test testsharder.Test, tester Tester, outputDir string) (*runtests.TestDetails, error) {
diff --git a/go.mod b/go.mod
index 48997a7..cd44254 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 // indirect
github.com/kr/fs v0.1.0 // indirect
+ github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b
github.com/pkg/errors v0.8.1 // indirect
github.com/pkg/sftp v1.8.3
go.chromium.org/luci v0.0.0-20181218015242-20acb618582d
diff --git a/go.sum b/go.sum
index 49189fa..0490d22 100644
--- a/go.sum
+++ b/go.sum
@@ -23,6 +23,8 @@
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b h1:ioAf++SXYsNBCJQdUQYpKsGUFUGvbxHaXFEnonuitI8=
+github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
