tools/testing/testrunner/outputs_test.go

// Copyright 2020 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testrunner

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"

	"go.fuchsia.dev/fuchsia/tools/lib/osmisc"
	"go.fuchsia.dev/fuchsia/tools/testing/runtests"
	"go.fuchsia.dev/fuchsia/tools/testing/tap"
)

func writeFiles(dir string, pathsToContents map[string]string) error {
	for p, contents := range pathsToContents {
		f, err := osmisc.CreateFile(filepath.Join(dir, p))
		if err != nil {
			return err
		}
		defer f.Close()
		if _, err := f.Write([]byte(contents)); err != nil {
			return err
		}
	}
	return nil
}

func TestRecordingOfOutputs(t *testing.T) {
	start := time.Unix(0, 0)
	testOutputDir := t.TempDir()
	suiteOutputDir := "suite_outputs"
	suiteOutputFile1 := filepath.Join(suiteOutputDir, "file1")
	suiteOutputFile2 := filepath.Join(suiteOutputDir, "file2")
	caseOutputFile := "case_outputs"
	origOutputs := map[string]string{
		suiteOutputFile1: "test outputs",
		suiteOutputFile2: "test outputs 2",
		caseOutputFile:   "testcase outputs",
	}
	if err := writeFiles(testOutputDir, origOutputs); err != nil {
		t.Errorf("failed to write output files: %s", err)
	}
	results := []TestResult{
		{
			Name:      "fuchsia-pkg://foo#test_a",
			GNLabel:   "//a/b/c:test_a(//toolchain)",
			Result:    runtests.TestFailure,
			StartTime: start,
			EndTime:   start.Add(5 * time.Millisecond),
			DataSinks: runtests.DataSinkReference{
				Sinks: runtests.DataSinkMap{
					"sinks": []runtests.DataSink{
						{
							Name: "SINK_A1",
							File: "sink_a1.txt",
						},
						{
							Name: "SINK_A2",
							File: "sink_a2.txt",
						},
					},
				},
			},
			Cases: []runtests.TestCaseResult{
				{
					DisplayName: "case1",
					CaseName:    "case1",
					Status:      runtests.TestFailure,
					Format:      "FTF",
					// Test having the OutputFile be a filename.
					OutputFiles: []string{caseOutputFile},
					OutputDir:   testOutputDir,
				},
			},
			// Test having the OutputFile be a directory name.
			OutputFiles: []string{suiteOutputDir},
			OutputDir:   testOutputDir,
			Stdio:       []byte("STDOUT_A"),
		},
		{
			Name:      "test_b",
			GNLabel:   "//a/b/c:test_b(//toolchain)",
			Result:    runtests.TestSuccess,
			StartTime: start,
			EndTime:   start.Add(10 * time.Millisecond),
			Stdio:     []byte("STDERR_B"),
		},
	}

	dataDir := t.TempDir()
	outDir := filepath.Join(dataDir, "out")

	var buf bytes.Buffer
	producer := tap.NewProducer(&buf)
	producer.Plan(len(results))
	o, err := CreateTestOutputs(producer, outDir)
	if err != nil {
		t.Fatal(err)
	}
	defer o.Close()

	outputFileA := func(filename string) string {
		return filepath.Join(url.PathEscape("fuchsia-pkg//foo#test_a"), "0", filename)
	}

	outputFileB := func(filename string) string {
		return filepath.Join("test_b", "0", filename)
	}

	testAStdout := outputFileA("stdout-and-stderr.txt")
	testASuiteOutputFile1 := outputFileA(suiteOutputFile1)
	testASuiteOutputFile2 := outputFileA(suiteOutputFile2)
	testACaseOutputFile := outputFileA(filepath.Join("case1", caseOutputFile))
	testBStdout := outputFileB("stdout-and-stderr.txt")
	expectedSummary := runtests.TestSummary{
		Tests: []runtests.TestDetails{{
			Name:    "fuchsia-pkg://foo#test_a",
			GNLabel: "//a/b/c:test_a(//toolchain)",
			// The expected OutputFiles in TestSummary should contain only files.
			// If any of the TestResult OutputFiles point to directories, TestOutputs.Record()
			// should list out all the files in those directories here.
			OutputFiles:    []string{testASuiteOutputFile1, testASuiteOutputFile2, testAStdout},
			Result:         runtests.TestFailure,
			StartTime:      start,
			DurationMillis: 5,
			DataSinks: runtests.DataSinkMap{
				"sinks": []runtests.DataSink{
					{
						Name: "SINK_A1",
						File: "sink_a1.txt",
					},
					{
						Name: "SINK_A2",
						File: "sink_a2.txt",
					},
				},
			},
			Cases: []runtests.TestCaseResult{
				{
					DisplayName: "case1",
					CaseName:    "case1",
					Status:      runtests.TestFailure,
					Format:      "FTF",
					OutputFiles: []string{testACaseOutputFile},
				},
			},
		}, {
			Name:           "test_b",
			GNLabel:        "//a/b/c:test_b(//toolchain)",
			OutputFiles:    []string{testBStdout},
			Result:         runtests.TestSuccess,
			StartTime:      start,
			DurationMillis: 10,
			// The data sinks will be added through a call to updateDataSinks().
			DataSinks: runtests.DataSinkMap{
				"sinks": []runtests.DataSink{
					{
						Name: "SINK_B",
						File: "other_dir/sink_b.txt",
					},
				},
			},
		}},
	}

	summaryBytes, err := json.Marshal(&expectedSummary)
	if err != nil {
		t.Fatalf("failed to marshal expected summary: %v", err)
	}

	expectedSinks := map[string]string{
		"sink_a1.txt": "SINK_A1",
		"sink_a2.txt": "SINK_A2",
		"sink_b.txt":  "SINK_B",
	}

	// Populate all of the expected output files.
	expectedContents := map[string]string{
		testAStdout:           "STDOUT_A",
		testBStdout:           "STDERR_B",
		testASuiteOutputFile1: origOutputs[suiteOutputFile1],
		testASuiteOutputFile2: origOutputs[suiteOutputFile2],
		testACaseOutputFile:   origOutputs[caseOutputFile],
		"summary.json":        string(summaryBytes),
	}
	for name, content := range expectedSinks {
		// Add sinks to expectedContents.
		expectedContents[name] = content
		path := filepath.Join(o.OutDir, name)
		dir := filepath.Dir(path)
		if err := os.MkdirAll(dir, 0o700); err != nil {
			t.Fatalf("failed to make directory %q for outputs: %v", dir, err)
		}
		if err := ioutil.WriteFile(path, []byte(content), 0o400); err != nil {
			t.Fatalf("failed to write contents %q to file %q: %v", content, name, err)
		}
	}

	for _, result := range results {
		if err := o.Record(context.Background(), result); err != nil {
			t.Fatalf("failed to record result of %q: %v", result.Name, err)
		}
	}
	o.updateDataSinks(map[string]runtests.DataSinkReference{
		"test_b": {
			Sinks: runtests.DataSinkMap{
				"sinks": []runtests.DataSink{
					{
						Name: "SINK_B",
						File: "sink_b.txt",
					},
				},
			},
		},
	}, "other_dir")
	o.Close()

	// Verify that the summary as expected.
	actualSummary := o.Summary
	if diff := cmp.Diff(expectedSummary, actualSummary); diff != "" {
		t.Errorf("Diff in test summary (-want +got):\n%s", diff)
	}

	// Verify that the TAP output is as expected.
	expectedTAPOutput := strings.TrimSpace(`
TAP version 13
1..2
not ok 1 fuchsia-pkg://foo#test_a (5ms)
ok 2 test_b (10ms)
`)
	actualTAPOutput := strings.TrimSpace(buf.String())
	if diff := cmp.Diff(expectedTAPOutput, actualTAPOutput); diff != "" {
		t.Errorf("TAP output diff (-want +got):\n%s", diff)
	}

	// Verify that the outDir's contents are as expected.
	outDirContents := make(map[string]string)
	for name := range expectedContents {
		path := filepath.Join(outDir, name)
		b, err := ioutil.ReadFile(path)
		if err != nil {
			t.Errorf("failed to read file %q in out dir: %v", path, err)
		}
		outDirContents[name] = string(b)
	}

	if diff := cmp.Diff(expectedContents, outDirContents); diff != "" {
		t.Errorf("Diff in out dir contents (-want +got):\n%s", diff)
	}
}