cmd/update_test_durations/update.go - infra/infra - Git at Google

 // Copyright 2022 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 // This tool queries ResultDB data from BigQuery to retrieve statistics about to
 // the durations of Fuchsia tests. The resulting data will be uploaded to CIPD
 // for use by the testsharder tool:
 // https://fuchsia.googlesource.com/fuchsia/+/HEAD/tools/integration/testsharder

 package main

 import (
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"time"

 	"github.com/maruel/subcommands"
 	"go.chromium.org/luci/auth"
 	"go.fuchsia.dev/infra/functools"
 )

 var errZeroTotalRuns = errors.New("cannot calculate average duration for tests with zero total runs")

 const (
 	// For each set of duration files (public and internal), the statistics for
 	// all tests will be aggregated to create a new file containing each test's
 	// average duration across all builders. This file will be used by any
 	// builders that don't yet have a corresponding duration file.
 	defaultBuilderName = "default"

 	// Every duration file will have a default entry that's applied to any test
 	// that doesn't yet have an entry (probably because it is new or renamed). It
 	// will have this value in its "name" field. We chose "*" to strike a balance
 	// between distinguishing this entry from actual tests (text "default" would
 	// be easier to mistake for an actual test) and being obviously intentional
 	// and clear about its purpose (an empty string would less clearly be a
 	// fallback, and might even suggest a bug in the updater).
 	defaultTestName = "*"

 	// How long to keep around old entries in test duration files before
 	// deleting them. Old entries are kept around for some time so we still have
 	// data in case a builder or test gets deleted and then reintroduced within
 	// the time window.
 	garbageCollectionAge = 24 * 30 * time.Hour
 )

 func cmdRun(authOpts auth.Options) *subcommands.Command {
 	return &subcommands.Command{
 		UsageLine: "run -dir DIR -project PROJECT",
 		ShortDesc: "write updated test duration data to a directory",
 		LongDesc:  "write updated test duration data to a directory",
 		CommandRun: func() subcommands.CommandRun {
 			c := &runCmd{}
 			c.Init(authOpts)
 			return c
 		},
 	}
 }

 type runCmd struct {
 	commonFlags

 	previousVersionDir string
 	outputDir          string
 	luciProject        string
 	dataWindowDays     int
 }

 // test is a pairing of a test and builder, along with the number of recorded
 // runs and median duration within the time window. The order of the fields here
 // must be kept in sync with the order of the rows returned by the BigQuery
 // query.
 type test struct {
 	Name string `json:"name"`

 	// The number of test runs included in calculating the duration.
 	Runs int64 `json:"runs"`

 	// The median duration of the test, in milliseconds, across all included runs.
 	MedianDurationMS int64 `json:"median_duration_ms"`

 	// The builder that ran this test.
 	//
 	// Test durations are separated into files by builder, so it's not necessary
 	// to include the builder name in the output JSON.
 	Builder string `json:"-"`

 	// The timestamp of the last test duration update that included new data for
 	// this test. Used for garbage-collecting old duration data.
 	LastSeen time.Time `json:"last_seen"`
 }

 // testDurationsMap maps from the name of a builder to a list of tests that were
 // run by that builder. Each entry in the map corresponds to one file in the
 // resulting test duration files.
 type testDurationMap map[string][]test

 func (c *runCmd) Init(defaultAuthOpts auth.Options) {
 	c.commonFlags.Init(defaultAuthOpts)
 	c.Flags.StringVar(
 		&c.previousVersionDir,
 		"previous-version-dir",
 		"",
 		"Directory containing the previous version of the durations files, which "+
 			"will be merged with the updated durations.")
 	c.Flags.StringVar(
 		&c.outputDir,
 		"output-dir",
 		"",
 		"Directory into which final duration files should be written.")
 	c.Flags.StringVar(
 		&c.luciProject,
 		"project",
 		"fuchsia",
 		"LUCI project to query test durations for.")
 	c.Flags.IntVar(
 		&c.dataWindowDays,
 		"days",
 		3,
 		"LUCI project to query test durations for.")
 }

 func (c *runCmd) parseArgs([]string) error {
 	return c.commonFlags.Parse()
 }

 func (c *runCmd) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
 	if err := c.parseArgs(args); err != nil {
 		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
 		return 1
 	}

 	if err := c.main(context.Background()); err != nil {
 		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
 		return 1
 	}
 	return 0
 }

 // updateTestDurations fetches the latest test durations from BigQuery and
 // uploads them to CIPD.
 func (c *runCmd) main(ctx context.Context) error {
 	if c.previousVersionDir == "" {
 		return errors.New("flag -previous-version-dir is required")
 	}
 	if c.outputDir == "" {
 		return errors.New("flag -output-dir is required")
 	}
 	fetchCtx, cancel := context.WithTimeout(ctx, queryTimeout)
 	defer cancel()
 	rows, err := queryLatestTestDurations(fetchCtx, c.luciProject, c.dataWindowDays)
 	if err != nil {
 		return err
 	}

 	date := time.Now().UTC().Round(24 * time.Hour)
 	return update(rows, c.previousVersionDir, c.outputDir, false, date)
 }

 func update(newData []test, previousVersionDir string, outputDir string, indentOutput bool, date time.Time) error {
 	oldDurations, err := unmarshalDurations(previousVersionDir, date)
 	if err != nil {
 		return err
 	}

 	newDurations, err := splitTestsByBuilder(newData)
 	if err != nil {
 		return err
 	}

 	for builder := range newDurations {
 		for i := range newDurations[builder] {
 			newDurations[builder][i].LastSeen = date
 		}
 	}

 	for builder, oldTests := range oldDurations {
 		newTestNames := make(map[string]struct{})
 		for _, newTest := range newDurations[builder] {
 			newTestNames[newTest.Name] = struct{}{}
 		}

 		for _, t := range oldTests {
 			if t.LastSeen.IsZero() {
 				// For backwards compatibility with files from before the
 				// LastSeen field was added.
 				t.LastSeen = date
 			}
 			if date.Sub(t.LastSeen) > garbageCollectionAge {
 				continue
 			}
 			if _, ok := newTestNames[t.Name]; ok {
 				// Ignore old data if there is new data for the test.
 				continue
 			}
 			newDurations[builder] = append(newDurations[builder], t)
 		}
 	}

 	defaultDurations, err := calculateDefaultDurations(newDurations)
 	if err != nil {
 		return err
 	}
 	newDurations[defaultBuilderName] = defaultDurations

 	if err := addDefaultEntries(newDurations); err != nil {
 		return err
 	}

 	return writeDurations(newDurations, outputDir, indentOutput)
 }

 func unmarshalDurations(dir string, date time.Time) (testDurationMap, error) {
 	files, err := os.ReadDir(dir)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read directory: %w", err)
 	}
 	res := make(testDurationMap)
 	for _, f := range files {
 		if f.IsDir() || !strings.HasSuffix(f.Name(), ".json") {
 			continue
 		}
 		builderName := strings.TrimSuffix(f.Name(), ".json")
 		if builderName == defaultBuilderName {
 			// Don't bother deserializing the default builder data, we'll
 			// recompute it from scratch based on up-to-date data.
 			continue
 		}
 		b, err := os.ReadFile(filepath.Join(dir, f.Name()))
 		if err != nil {
 			return nil, fmt.Errorf("failed to read file: %w", err)
 		}
 		var tests []test
 		if err := json.Unmarshal(b, &tests); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal file %s: %w", f.Name(), err)
 		}
 		for _, test := range tests {
 			// Don't bother deserializing the default test data, we'll recompute
 			// it from scratch based on up-to-date data.
 			if test.Name == defaultTestName {
 				continue
 			}
 			if test.LastSeen.IsZero() {
 				test.LastSeen = date
 			}
 			res[builderName] = append(res[builderName], test)
 		}
 	}
 	return res, nil
 }

 // splitTestsByBuilder takes a collection of tests and arranges them into a
 // mapping corresponding to the data that should be written to each per-builder
 // test duration file.
 func splitTestsByBuilder(tests []test) (testDurationMap, error) {
 	// Mapping from builder name to list of tests.
 	durations := make(testDurationMap)

 	for _, t := range tests {
 		if t.Builder == "" {
 			return nil, fmt.Errorf("test has empty builder: %+v", t)
 		}
 		durations[t.Builder] = append(durations[t.Builder], t)
 	}

 	return durations, nil
 }

 // calculateDefaultDurations aggregates durations for tests across all builders.
 // testsharder will use this default file for any unknown builder that doesn't
 // yet have its own durations file.
 //
 // We use the average duration of all tests, rather than the median, so that
 // when using this default file, the sum of all tests' expected durations is
 // close to the sum of actual durations. If we used the median of all durations,
 // the sum of expected durations would likely be far too low because it wouldn't
 // account for the long tail of slower tests, and we would end up producing too
 // few shards when executing testsharder with a target per-shard duration.
 //
 // See unit tests for examples.
 func calculateDefaultDurations(durations testDurationMap) ([]test, error) {
 	testsByName := make(map[string][]test)
 	for _, tests := range durations {
 		for _, t := range tests {
 			testsByName[t.Name] = append(testsByName[t.Name], t)
 		}
 	}

 	var defaultDurations []test
 	for name, sameNameTests := range testsByName {
 		var totalRuns int64
 		for _, t := range sameNameTests {
 			totalRuns += t.Runs
 		}

 		duration, err := averageDurationMS(sameNameTests)
 		if err != nil {
 			return nil, err
 		}

 		defaultDurations = append(defaultDurations, test{
 			Name:             name,
 			Builder:          defaultBuilderName,
 			Runs:             totalRuns,
 			MedianDurationMS: duration,
 		})
 	}

 	// The SQL query ensures that all other builders' tests are sorted, so we
 	// sort these too for consistency.
 	sort.Slice(defaultDurations, func(i, j int) bool {
 		return defaultDurations[i].Name < defaultDurations[j].Name
 	})

 	return defaultDurations, nil
 }

 // addDefaultEntries adds an new entry for each builder at index zero that
 // will be applied by testsharder to any test that doesn't have its own entry,
 // probably because it's a new test or has been renamed.
 //
 // The average of all existing tests' median durations is probably a good
 // estimate of the duration for any such tests. Average of medians is better
 // than median of medians in the case where many new tests are added, assuming
 // that the distribution of durations of the new tests is similar to the
 // distribution for existing tests. (If only a couple tests are added, it
 // doesn't make much of a difference either way). The median of medians might be
 // closer to the actual duration of *most* of the new tests, but it doesn't take
 // into account the fact that a few of the new tests are probably much longer
 // than the rest. This would lead to all the new tests being put into the same
 // shard, which would always time out because it has too many long tests.
 func addDefaultEntries(durations testDurationMap) error {
 	for builder, builderTests := range durations {
 		defaultDuration, err := averageDurationMS(builderTests)
 		if err != nil {
 			return err
 		}
 		defaultTestEntry := test{
 			Name:             defaultTestName,
 			Builder:          builder,
 			MedianDurationMS: defaultDuration,
 		}
 		durations[builder] = append([]test{defaultTestEntry}, builderTests...)
 	}
 	return nil
 }

 // averageDurationMS calculates the average of median durations, weighted by
 // runs, for all the given tests.
 func averageDurationMS(tests []test) (int64, error) {
 	var totalDurationMS, totalRuns int64
 	for _, t := range tests {
 		totalRuns += t.Runs
 		totalDurationMS += t.MedianDurationMS * t.Runs
 	}
 	if totalRuns == 0 {
 		// This likely indicates a bug somewhere in the SQL query or Go code.
 		return 0, fmt.Errorf("%w: %+v", errZeroTotalRuns, tests)
 	}
 	avg := float64(totalDurationMS) / float64(totalRuns)
 	return int64(math.Round(avg)), nil
 }

 func writeDurations(durations testDurationMap, dir string, indentOutput bool) error {
 	for builder, tests := range durations {
 		functools.SortBy(tests, func(t test) string {
 			return t.Name
 		})
 		var b []byte
 		var err error
 		if indentOutput {
 			b, err = json.MarshalIndent(tests, "", "  ")
 		} else {
 			b, err = json.Marshal(tests)
 		}
 		if err != nil {
 			return err
 		}

 		if indentOutput {
 			b = append(b, '\n')
 		}
 		fileName := fmt.Sprintf("%s.json", builder)
 		if err := os.WriteFile(filepath.Join(dir, fileName), b, 0o600); err != nil {
 			return err
 		}
 	}
 	return nil
 }
	// Copyright 2022 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	// This tool queries ResultDB data from BigQuery to retrieve statistics about to
	// the durations of Fuchsia tests. The resulting data will be uploaded to CIPD
	// for use by the testsharder tool:
	// https://fuchsia.googlesource.com/fuchsia/+/HEAD/tools/integration/testsharder

	package main

	import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/maruel/subcommands"
	"go.chromium.org/luci/auth"
	"go.fuchsia.dev/infra/functools"
	)

	var errZeroTotalRuns = errors.New("cannot calculate average duration for tests with zero total runs")

	const (
	// For each set of duration files (public and internal), the statistics for
	// all tests will be aggregated to create a new file containing each test's
	// average duration across all builders. This file will be used by any
	// builders that don't yet have a corresponding duration file.
	defaultBuilderName = "default"

	// Every duration file will have a default entry that's applied to any test
	// that doesn't yet have an entry (probably because it is new or renamed). It
	// will have this value in its "name" field. We chose "*" to strike a balance
	// between distinguishing this entry from actual tests (text "default" would
	// be easier to mistake for an actual test) and being obviously intentional
	// and clear about its purpose (an empty string would less clearly be a
	// fallback, and might even suggest a bug in the updater).
	defaultTestName = "*"

	// How long to keep around old entries in test duration files before
	// deleting them. Old entries are kept around for some time so we still have
	// data in case a builder or test gets deleted and then reintroduced within
	// the time window.
	garbageCollectionAge = 24 * 30 * time.Hour
	)

	func cmdRun(authOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
	UsageLine: "run -dir DIR -project PROJECT",
	ShortDesc: "write updated test duration data to a directory",
	LongDesc: "write updated test duration data to a directory",
	CommandRun: func() subcommands.CommandRun {
	c := &runCmd{}
	c.Init(authOpts)
	return c
	},
	}
	}

	type runCmd struct {
	commonFlags

	previousVersionDir string
	outputDir string
	luciProject string
	dataWindowDays int
	}

	// test is a pairing of a test and builder, along with the number of recorded
	// runs and median duration within the time window. The order of the fields here
	// must be kept in sync with the order of the rows returned by the BigQuery
	// query.
	type test struct {
	Name string `json:"name"`

	// The number of test runs included in calculating the duration.
	Runs int64 `json:"runs"`

	// The median duration of the test, in milliseconds, across all included runs.
	MedianDurationMS int64 `json:"median_duration_ms"`

	// The builder that ran this test.
	//
	// Test durations are separated into files by builder, so it's not necessary
	// to include the builder name in the output JSON.
	Builder string `json:"-"`

	// The timestamp of the last test duration update that included new data for
	// this test. Used for garbage-collecting old duration data.
	LastSeen time.Time `json:"last_seen"`
	}

	// testDurationsMap maps from the name of a builder to a list of tests that were
	// run by that builder. Each entry in the map corresponds to one file in the
	// resulting test duration files.
	type testDurationMap map[string][]test

	func (c *runCmd) Init(defaultAuthOpts auth.Options) {
	c.commonFlags.Init(defaultAuthOpts)
	c.Flags.StringVar(
	&c.previousVersionDir,
	"previous-version-dir",
	"",
	"Directory containing the previous version of the durations files, which "+
	"will be merged with the updated durations.")
	c.Flags.StringVar(
	&c.outputDir,
	"output-dir",
	"",
	"Directory into which final duration files should be written.")
	c.Flags.StringVar(
	&c.luciProject,
	"project",
	"fuchsia",
	"LUCI project to query test durations for.")
	c.Flags.IntVar(
	&c.dataWindowDays,
	"days",
	3,
	"LUCI project to query test durations for.")
	}

	func (c *runCmd) parseArgs([]string) error {
	return c.commonFlags.Parse()
	}

	func (c *runCmd) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.parseArgs(args); err != nil {
	fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
	return 1
	}

	if err := c.main(context.Background()); err != nil {
	fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
	return 1
	}
	return 0
	}

	// updateTestDurations fetches the latest test durations from BigQuery and
	// uploads them to CIPD.
	func (c *runCmd) main(ctx context.Context) error {
	if c.previousVersionDir == "" {
	return errors.New("flag -previous-version-dir is required")
	}
	if c.outputDir == "" {
	return errors.New("flag -output-dir is required")
	}
	fetchCtx, cancel := context.WithTimeout(ctx, queryTimeout)
	defer cancel()
	rows, err := queryLatestTestDurations(fetchCtx, c.luciProject, c.dataWindowDays)
	if err != nil {
	return err
	}

	date := time.Now().UTC().Round(24 * time.Hour)
	return update(rows, c.previousVersionDir, c.outputDir, false, date)
	}

	func update(newData []test, previousVersionDir string, outputDir string, indentOutput bool, date time.Time) error {
	oldDurations, err := unmarshalDurations(previousVersionDir, date)
	if err != nil {
	return err
	}

	newDurations, err := splitTestsByBuilder(newData)
	if err != nil {
	return err
	}

	for builder := range newDurations {
	for i := range newDurations[builder] {
	newDurations[builder][i].LastSeen = date
	}
	}

	for builder, oldTests := range oldDurations {
	newTestNames := make(map[string]struct{})
	for _, newTest := range newDurations[builder] {
	newTestNames[newTest.Name] = struct{}{}
	}

	for _, t := range oldTests {
	if t.LastSeen.IsZero() {
	// For backwards compatibility with files from before the
	// LastSeen field was added.
	t.LastSeen = date
	}
	if date.Sub(t.LastSeen) > garbageCollectionAge {
	continue
	}
	if _, ok := newTestNames[t.Name]; ok {
	// Ignore old data if there is new data for the test.
	continue
	}
	newDurations[builder] = append(newDurations[builder], t)
	}
	}

	defaultDurations, err := calculateDefaultDurations(newDurations)
	if err != nil {
	return err
	}
	newDurations[defaultBuilderName] = defaultDurations

	if err := addDefaultEntries(newDurations); err != nil {
	return err
	}

	return writeDurations(newDurations, outputDir, indentOutput)
	}

	func unmarshalDurations(dir string, date time.Time) (testDurationMap, error) {
	files, err := os.ReadDir(dir)
	if err != nil {
	return nil, fmt.Errorf("failed to read directory: %w", err)
	}
	res := make(testDurationMap)
	for _, f := range files {
	if f.IsDir() \|\| !strings.HasSuffix(f.Name(), ".json") {
	continue
	}
	builderName := strings.TrimSuffix(f.Name(), ".json")
	if builderName == defaultBuilderName {
	// Don't bother deserializing the default builder data, we'll
	// recompute it from scratch based on up-to-date data.
	continue
	}
	b, err := os.ReadFile(filepath.Join(dir, f.Name()))
	if err != nil {
	return nil, fmt.Errorf("failed to read file: %w", err)
	}
	var tests []test
	if err := json.Unmarshal(b, &tests); err != nil {
	return nil, fmt.Errorf("failed to unmarshal file %s: %w", f.Name(), err)
	}
	for _, test := range tests {
	// Don't bother deserializing the default test data, we'll recompute
	// it from scratch based on up-to-date data.
	if test.Name == defaultTestName {
	continue
	}
	if test.LastSeen.IsZero() {
	test.LastSeen = date
	}
	res[builderName] = append(res[builderName], test)
	}
	}
	return res, nil
	}

	// splitTestsByBuilder takes a collection of tests and arranges them into a
	// mapping corresponding to the data that should be written to each per-builder
	// test duration file.
	func splitTestsByBuilder(tests []test) (testDurationMap, error) {
	// Mapping from builder name to list of tests.
	durations := make(testDurationMap)

	for _, t := range tests {
	if t.Builder == "" {
	return nil, fmt.Errorf("test has empty builder: %+v", t)
	}
	durations[t.Builder] = append(durations[t.Builder], t)
	}

	return durations, nil
	}

	// calculateDefaultDurations aggregates durations for tests across all builders.
	// testsharder will use this default file for any unknown builder that doesn't
	// yet have its own durations file.
	//
	// We use the average duration of all tests, rather than the median, so that
	// when using this default file, the sum of all tests' expected durations is
	// close to the sum of actual durations. If we used the median of all durations,
	// the sum of expected durations would likely be far too low because it wouldn't
	// account for the long tail of slower tests, and we would end up producing too
	// few shards when executing testsharder with a target per-shard duration.
	//
	// See unit tests for examples.
	func calculateDefaultDurations(durations testDurationMap) ([]test, error) {
	testsByName := make(map[string][]test)
	for _, tests := range durations {
	for _, t := range tests {
	testsByName[t.Name] = append(testsByName[t.Name], t)
	}
	}

	var defaultDurations []test
	for name, sameNameTests := range testsByName {
	var totalRuns int64
	for _, t := range sameNameTests {
	totalRuns += t.Runs
	}

	duration, err := averageDurationMS(sameNameTests)
	if err != nil {
	return nil, err
	}

	defaultDurations = append(defaultDurations, test{
	Name: name,
	Builder: defaultBuilderName,
	Runs: totalRuns,
	MedianDurationMS: duration,
	})
	}

	// The SQL query ensures that all other builders' tests are sorted, so we
	// sort these too for consistency.
	sort.Slice(defaultDurations, func(i, j int) bool {
	return defaultDurations[i].Name < defaultDurations[j].Name
	})

	return defaultDurations, nil
	}

	// addDefaultEntries adds an new entry for each builder at index zero that
	// will be applied by testsharder to any test that doesn't have its own entry,
	// probably because it's a new test or has been renamed.
	//
	// The average of all existing tests' median durations is probably a good
	// estimate of the duration for any such tests. Average of medians is better
	// than median of medians in the case where many new tests are added, assuming
	// that the distribution of durations of the new tests is similar to the
	// distribution for existing tests. (If only a couple tests are added, it
	// doesn't make much of a difference either way). The median of medians might be
	// closer to the actual duration of most of the new tests, but it doesn't take
	// into account the fact that a few of the new tests are probably much longer
	// than the rest. This would lead to all the new tests being put into the same
	// shard, which would always time out because it has too many long tests.
	func addDefaultEntries(durations testDurationMap) error {
	for builder, builderTests := range durations {
	defaultDuration, err := averageDurationMS(builderTests)
	if err != nil {
	return err
	}
	defaultTestEntry := test{
	Name: defaultTestName,
	Builder: builder,
	MedianDurationMS: defaultDuration,
	}
	durations[builder] = append([]test{defaultTestEntry}, builderTests...)
	}
	return nil
	}

	// averageDurationMS calculates the average of median durations, weighted by
	// runs, for all the given tests.
	func averageDurationMS(tests []test) (int64, error) {
	var totalDurationMS, totalRuns int64
	for _, t := range tests {
	totalRuns += t.Runs
	totalDurationMS += t.MedianDurationMS * t.Runs
	}
	if totalRuns == 0 {
	// This likely indicates a bug somewhere in the SQL query or Go code.
	return 0, fmt.Errorf("%w: %+v", errZeroTotalRuns, tests)
	}
	avg := float64(totalDurationMS) / float64(totalRuns)
	return int64(math.Round(avg)), nil
	}

	func writeDurations(durations testDurationMap, dir string, indentOutput bool) error {
	for builder, tests := range durations {
	functools.SortBy(tests, func(t test) string {
	return t.Name
	})
	var b []byte
	var err error
	if indentOutput {
	b, err = json.MarshalIndent(tests, "", " ")
	} else {
	b, err = json.Marshal(tests)
	}
	if err != nil {
	return err
	}

	if indentOutput {
	b = append(b, '\n')
	}
	fileName := fmt.Sprintf("%s.json", builder)
	if err := os.WriteFile(filepath.Join(dir, fileName), b, 0o600); err != nil {
	return err
	}
	}
	return nil
	}