cmd/update_test_durations/update_test.go

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
)

var (
	updateGoldens = flag.Bool("update-goldens", false, "Whether to update goldens")
)

func TestUpdate(t *testing.T) {
	t.Parallel()

	newData := []test{
		{
			Name:             "still_existing_test",
			Builder:          "foo",
			MedianDurationMS: 10,
			Runs:             3,
		},
		{
			Name:             "still_existing_test",
			Builder:          "new-builder",
			MedianDurationMS: 100,
			Runs:             12,
		},
		{
			Name:             "new_test",
			Builder:          "foo",
			MedianDurationMS: 12,
			Runs:             4,
		},
	}

	goldensDir := "testdata/goldens"

	var outputDir string
	if *updateGoldens {
		// Clear the goldens directory to garbage-collect old files.
		if err := os.RemoveAll(goldensDir); err != nil {
			t.Fatal(err)
		}
		if err := os.MkdirAll(goldensDir, 0o700); err != nil {
			t.Fatal(err)
		}
		outputDir = goldensDir
	} else {
		outputDir = t.TempDir()
	}

	now := time.Date(2023, 11, 23, 0, 0, 0, 0, time.UTC)
	if err := update(newData, "testdata/old_durations", outputDir, true, now); err != nil {
		t.Fatalf("Unexpected error: %s", err)
	}

	// We directly wrote the files to the goldens directory, so we know they'll
	// be up-to-date, no point in checking them.
	if *updateGoldens {
		return
	}

	cmd := exec.Command(
		"git", "diff", "--no-index", "--no-ext-diff", "--no-prefix",
		goldensDir, outputDir,
	)
	var stdout, stderr strings.Builder
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err := cmd.Run()
	var errExit *exec.ExitError
	if errors.As(err, &errExit) && errExit.ExitCode() == 1 {
		t.Fatalf(
			"Unexpected diff: %s\n\n"+
				"To update goldens, run: `go test ./cmd/update_test_durations -update-goldens`",
			stdout.String())
	} else if err != nil {
		t.Fatalf("Error running git diff: %s", err)
	}
}

func TestCalculateDefaultDurations(t *testing.T) {
	t.Parallel()

	durations := testDurationMap{
		"bar": append(
			testsWithDurations("bar", 3),
			test{Name: "shared", Runs: 2, MedianDurationMS: 20}),
		"foo": append(
			testsWithDurations("foo", 1),
			test{Name: "shared", Runs: 1, MedianDurationMS: 5}),
	}

	got, err := calculateDefaultDurations(durations)
	if err != nil {
		t.Fatal(err)
	}

	want := appendAll(
		testsWithDurations("bar", 3),
		testsWithDurations("foo", 1),
		[]test{
			{
				Name:             "shared",
				Runs:             3,
				MedianDurationMS: 15,
			},
		})
	if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(test{}, "Builder")); diff != "" {
		t.Errorf("Unexpected diff:\n%s", diff)
	}
}

// testsWithDurations constructs a slice of tests with the given durations, in
// order, all have the same builder.
func testsWithDurations(builder string, durations ...int64) []test {
	var res []test
	for i, duration := range durations {
		res = append(res, test{
			Name:             fmt.Sprintf("%s-%d", builder, i),
			Builder:          builder,
			MedianDurationMS: duration,
			Runs:             1,
		})
	}
	return res
}

func appendAll(slices ...[]test) []test {
	var res []test
	for _, s := range slices {
		res = append(res, s...)
	}
	return res
}