tools/testing/testrunner/cmd/main.go

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"

	build "go.fuchsia.dev/fuchsia/tools/build/lib"
	"go.fuchsia.dev/fuchsia/tools/lib/color"
	"go.fuchsia.dev/fuchsia/tools/lib/logger"
	"go.fuchsia.dev/fuchsia/tools/net/sshutil"
	"go.fuchsia.dev/fuchsia/tools/testing/runtests"
	tap "go.fuchsia.dev/fuchsia/tools/testing/tap/lib"
	testparser "go.fuchsia.dev/fuchsia/tools/testing/testparser/lib"
	testrunner "go.fuchsia.dev/fuchsia/tools/testing/testrunner/lib"
	"go.fuchsia.dev/fuchsia/tools/testing/util"
)

// Fuchsia-specific environment variables possibly exposed to the testrunner.
const (
	nodenameEnvVar = "FUCHSIA_NODENAME"
	sshKeyEnvVar   = "FUCHSIA_SSH_KEY"
	// A directory that will be automatically isolated on completion of a task.
	testOutdirEnvVar = "FUCHSIA_TEST_OUTDIR"
)

// Command-line flags
var (
	// Whether to show Usage and exit.
	help bool

	// The path where a directory containing test results should be created.
	outDir string

	// Working directory of the local testing subprocesses.
	localWD string

	// Whether to use runtests when executing tests on fuchsia. If false, the
	// default will be run_test_component.
	useRuntests bool

	// Per-test timeout.
	perTestTimeout time.Duration
)

func usage() {
	fmt.Printf(`testrunner [flags] tests-file

Executes all tests found in the JSON [tests-file]
Fuchsia tests require both the nodename of the fuchsia instance and a private
SSH key corresponding to a authorized key to be set in the environment under
%s and %s respectively.
`, nodenameEnvVar, sshKeyEnvVar)
}

func init() {
	flag.BoolVar(&help, "help", false, "Whether to show Usage and exit.")
	flag.StringVar(&outDir, "out-dir", "", "Optional path where a directory containing test results should be created.")
	flag.StringVar(&localWD, "C", "", "Working directory of local testing subprocesses; if unset the current working directory will be used.")
	flag.BoolVar(&useRuntests, "use-runtests", false, "Whether to default to running fuchsia tests with runtests; if false, run_test_component will be used.")
	// TODO(fxb/36480): Support different timeouts for different tests.
	flag.DurationVar(&perTestTimeout, "per-test-timeout", 0, "Per-test timeout, applied to all tests. Ignored if <= 0.")
	flag.Usage = usage
}

func main() {
	flag.Parse()

	if help || flag.NArg() != 1 {
		flag.Usage()
		flag.PrintDefaults()
		return
	}

	l := logger.NewLogger(logger.DebugLevel, color.NewColor(color.ColorAuto), os.Stdout, os.Stderr, "testrunner ")
	ctx := logger.WithLogger(context.Background(), l)

	testsPath := flag.Arg(0)
	tests, err := loadTests(testsPath)
	if err != nil {
		log.Fatalf("failed to load tests from %q: %v", testsPath, err)
	}

	// Configure a test outputs object, responsible for producing TAP output,
	// recording data sinks, and archiving other test outputs.
	testOutdir := filepath.Join(os.Getenv(testOutdirEnvVar), outDir)
	if testOutdir == "" {
		var err error
		testOutdir, err = ioutil.TempDir("", "testrunner")
		if err != nil {
			log.Fatalf("failed to create a test output directory")
		}
	}
	logger.Debugf(ctx, "test output directory: %s", testOutdir)

	tapProducer := tap.NewProducer(os.Stdout)
	tapProducer.Plan(len(tests))
	outputs, err := createTestOutputs(tapProducer, testOutdir)
	if err != nil {
		log.Fatalf("failed to create test results object: %v", err)
	}
	defer outputs.Close()

	nodename := os.Getenv(nodenameEnvVar)
	sshKeyFile := os.Getenv(sshKeyEnvVar)

	if err := execute(ctx, tests, outputs, nodename, sshKeyFile); err != nil {
		log.Fatal(err)
	}
}

func loadTests(path string) ([]build.Test, error) {
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read %q: %w", path, err)
	}

	var tests []build.Test
	if err := json.Unmarshal(bytes, &tests); err != nil {
		return nil, fmt.Errorf("failed to unmarshal %q: %w", path, err)
	}

	return tests, nil
}

type tester interface {
	Test(context.Context, build.Test, io.Writer, io.Writer) (runtests.DataSinkReference, error)
	Close() error
	CopySinks(context.Context, []runtests.DataSinkReference) error
}

func execute(ctx context.Context, tests []build.Test, outputs *testOutputs, nodename, sshKeyFile string) error {
	var localTests, fuchsiaTests []build.Test
	for _, test := range tests {
		switch test.OS {
		case "fuchsia":
			fuchsiaTests = append(fuchsiaTests, test)
		case "linux":
			if runtime.GOOS != "linux" {
				return fmt.Errorf("cannot run linux tests when GOOS = %q", runtime.GOOS)
			}
			localTests = append(localTests, test)
		case "mac":
			if runtime.GOOS != "darwin" {
				return fmt.Errorf("cannot run mac tests when GOOS = %q", runtime.GOOS)
			}
			localTests = append(localTests, test)
		default:
			return fmt.Errorf("test %#v has unsupported OS: %q", test, test.OS)
		}
	}

	localEnv := append(os.Environ(),
		// Tell tests written in Rust to print stack on failures.
		"RUST_BACKTRACE=1",
	)
	localTester := newSubprocessTester(localWD, localEnv, perTestTimeout)
	if err := runTests(ctx, localTests, localTester, outputs); err != nil {
		return err
	}

	if len(fuchsiaTests) == 0 {
		return nil
	}

	var t tester
	var err error
	if sshKeyFile != "" {
		if nodename == "" {
			return fmt.Errorf("%s must be set", nodenameEnvVar)
		}
		t, err = newFuchsiaSSHTester(ctx, nodename, sshKeyFile, outputs.outDir, useRuntests, perTestTimeout)
	} else {
		// TODO(fxbug.dev/41930): create a serial test runner in this case.
		return fmt.Errorf("%s must be set", sshKeyEnvVar)
	}
	if err != nil {
		return fmt.Errorf("failed to initialize fuchsia tester: %v", err)
	}
	defer t.Close()

	return runTests(ctx, fuchsiaTests, t, outputs)
}

func runTests(ctx context.Context, tests []build.Test, t tester, outputs *testOutputs) error {
	var sinks []runtests.DataSinkReference
	for _, test := range tests {
		result, err := runTest(ctx, test, t)
		if errors.Is(err, sshutil.ConnectionError) {
			return err
		}
		if err := outputs.record(*result); err != nil {
			return err
		}
		sinks = append(sinks, result.DataSinks)
	}
	return t.CopySinks(ctx, sinks)
}

func runTest(ctx context.Context, test build.Test, t tester) (*testrunner.TestResult, error) {
	result := runtests.TestSuccess
	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)

	multistdout := io.MultiWriter(stdout, os.Stdout)
	multistderr := io.MultiWriter(stderr, os.Stderr)

	startTime := time.Now()
	dataSinks, err := t.Test(ctx, test, multistdout, multistderr)
	if err != nil {
		result = runtests.TestFailure
		logger.Errorf(ctx, err.Error())
		if errors.Is(err, sshutil.ConnectionError) {
			return nil, err
		}
	}

	endTime := time.Now()

	name := util.UniqueName(test)

	// If test is a multiplier test, the name should end in a number.
	// Re-append that to the result name.
	re := regexp.MustCompile(`\([0-9]+\)$`)
	index := re.FindString(test.Name)
	if index != "" {
		name += "-" + index
	}

	// Record the test details in the summary.
	return &testrunner.TestResult{
		Name:      name,
		GNLabel:   test.Label,
		Stdout:    stdout.Bytes(),
		Stderr:    stderr.Bytes(),
		Result:    result,
		Cases:     testparser.Parse(stdout.Bytes()),
		StartTime: startTime,
		EndTime:   endTime,
		DataSinks: dataSinks,
	}, nil
}