| // Copyright 2022 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| // This tool queries ResultDB data from BigQuery to retrieve statistics about to |
| // the durations of Fuchsia tests. The resulting data will be uploaded to CIPD |
| // for use by the testsharder tool: |
| // https://fuchsia.googlesource.com/fuchsia/+/HEAD/tools/integration/testsharder |
| |
| package main |
| |
| import ( |
| "context" |
| "encoding/json" |
| "errors" |
| "fmt" |
| "math" |
| "os" |
| "path/filepath" |
| "sort" |
| "strings" |
| "time" |
| |
| "github.com/maruel/subcommands" |
| "go.chromium.org/luci/auth" |
| "go.fuchsia.dev/infra/functools" |
| ) |
| |
| var errZeroTotalRuns = errors.New("cannot calculate average duration for tests with zero total runs") |
| |
| const ( |
| // For each set of duration files (public and internal), the statistics for |
| // all tests will be aggregated to create a new file containing each test's |
| // average duration across all builders. This file will be used by any |
| // builders that don't yet have a corresponding duration file. |
| defaultBuilderName = "default" |
| |
| // Every duration file will have a default entry that's applied to any test |
| // that doesn't yet have an entry (probably because it is new or renamed). It |
| // will have this value in its "name" field. We chose "*" to strike a balance |
| // between distinguishing this entry from actual tests (text "default" would |
| // be easier to mistake for an actual test) and being obviously intentional |
| // and clear about its purpose (an empty string would less clearly be a |
| // fallback, and might even suggest a bug in the updater). |
| defaultTestName = "*" |
| |
| // How long to keep around old entries in test duration files before |
| // deleting them. Old entries are kept around for some time so we still have |
| // data in case a builder or test gets deleted and then reintroduced within |
| // the time window. |
| garbageCollectionAge = 24 * 30 * time.Hour |
| ) |
| |
| func cmdRun(authOpts auth.Options) *subcommands.Command { |
| return &subcommands.Command{ |
| UsageLine: "run -dir DIR -project PROJECT", |
| ShortDesc: "write updated test duration data to a directory", |
| LongDesc: "write updated test duration data to a directory", |
| CommandRun: func() subcommands.CommandRun { |
| c := &runCmd{} |
| c.Init(authOpts) |
| return c |
| }, |
| } |
| } |
| |
| type runCmd struct { |
| commonFlags |
| |
| previousVersionDir string |
| outputDir string |
| luciProject string |
| dataWindowDays int |
| } |
| |
| // test is a pairing of a test and builder, along with the number of recorded |
| // runs and median duration within the time window. The order of the fields here |
| // must be kept in sync with the order of the rows returned by the BigQuery |
| // query. |
| type test struct { |
| Name string `json:"name"` |
| |
| // The number of test runs included in calculating the duration. |
| Runs int64 `json:"runs"` |
| |
| // The median duration of the test, in milliseconds, across all included runs. |
| MedianDurationMS int64 `json:"median_duration_ms"` |
| |
| // The builder that ran this test. |
| // |
| // Test durations are separated into files by builder, so it's not necessary |
| // to include the builder name in the output JSON. |
| Builder string `json:"-"` |
| |
| // The timestamp of the last test duration update that included new data for |
| // this test. Used for garbage-collecting old duration data. |
| LastSeen time.Time `json:"last_seen"` |
| } |
| |
| // testDurationsMap maps from the name of a builder to a list of tests that were |
| // run by that builder. Each entry in the map corresponds to one file in the |
| // resulting test duration files. |
| type testDurationMap map[string][]test |
| |
| func (c *runCmd) Init(defaultAuthOpts auth.Options) { |
| c.commonFlags.Init(defaultAuthOpts) |
| c.Flags.StringVar( |
| &c.previousVersionDir, |
| "previous-version-dir", |
| "", |
| "Directory containing the previous version of the durations files, which "+ |
| "will be merged with the updated durations.") |
| c.Flags.StringVar( |
| &c.outputDir, |
| "output-dir", |
| "", |
| "Directory into which final duration files should be written.") |
| c.Flags.StringVar( |
| &c.luciProject, |
| "project", |
| "fuchsia", |
| "LUCI project to query test durations for.") |
| c.Flags.IntVar( |
| &c.dataWindowDays, |
| "days", |
| 3, |
| "LUCI project to query test durations for.") |
| } |
| |
| func (c *runCmd) parseArgs([]string) error { |
| return c.commonFlags.Parse() |
| } |
| |
| func (c *runCmd) Run(a subcommands.Application, args []string, _ subcommands.Env) int { |
| if err := c.parseArgs(args); err != nil { |
| fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) |
| return 1 |
| } |
| |
| if err := c.main(context.Background()); err != nil { |
| fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) |
| return 1 |
| } |
| return 0 |
| } |
| |
| // updateTestDurations fetches the latest test durations from BigQuery and |
| // uploads them to CIPD. |
| func (c *runCmd) main(ctx context.Context) error { |
| if c.previousVersionDir == "" { |
| return errors.New("flag -previous-version-dir is required") |
| } |
| if c.outputDir == "" { |
| return errors.New("flag -output-dir is required") |
| } |
| fetchCtx, cancel := context.WithTimeout(ctx, queryTimeout) |
| defer cancel() |
| rows, err := queryLatestTestDurations(fetchCtx, c.luciProject, c.dataWindowDays) |
| if err != nil { |
| return err |
| } |
| |
| date := time.Now().UTC().Round(24 * time.Hour) |
| return update(rows, c.previousVersionDir, c.outputDir, false, date) |
| } |
| |
| func update(newData []test, previousVersionDir string, outputDir string, indentOutput bool, date time.Time) error { |
| oldDurations, err := unmarshalDurations(previousVersionDir, date) |
| if err != nil { |
| return err |
| } |
| |
| newDurations, err := splitTestsByBuilder(newData) |
| if err != nil { |
| return err |
| } |
| |
| for builder := range newDurations { |
| for i := range newDurations[builder] { |
| newDurations[builder][i].LastSeen = date |
| } |
| } |
| |
| for builder, oldTests := range oldDurations { |
| newTestNames := make(map[string]struct{}) |
| for _, newTest := range newDurations[builder] { |
| newTestNames[newTest.Name] = struct{}{} |
| } |
| |
| for _, t := range oldTests { |
| if t.LastSeen.IsZero() { |
| // For backwards compatibility with files from before the |
| // LastSeen field was added. |
| t.LastSeen = date |
| } |
| if date.Sub(t.LastSeen) > garbageCollectionAge { |
| continue |
| } |
| if _, ok := newTestNames[t.Name]; ok { |
| // Ignore old data if there is new data for the test. |
| continue |
| } |
| newDurations[builder] = append(newDurations[builder], t) |
| } |
| } |
| |
| defaultDurations, err := calculateDefaultDurations(newDurations) |
| if err != nil { |
| return err |
| } |
| newDurations[defaultBuilderName] = defaultDurations |
| |
| if err := addDefaultEntries(newDurations); err != nil { |
| return err |
| } |
| |
| return writeDurations(newDurations, outputDir, indentOutput) |
| } |
| |
| func unmarshalDurations(dir string, date time.Time) (testDurationMap, error) { |
| files, err := os.ReadDir(dir) |
| if err != nil { |
| return nil, fmt.Errorf("failed to read directory: %w", err) |
| } |
| res := make(testDurationMap) |
| for _, f := range files { |
| if f.IsDir() || !strings.HasSuffix(f.Name(), ".json") { |
| continue |
| } |
| builderName := strings.TrimSuffix(f.Name(), ".json") |
| if builderName == defaultBuilderName { |
| // Don't bother deserializing the default builder data, we'll |
| // recompute it from scratch based on up-to-date data. |
| continue |
| } |
| b, err := os.ReadFile(filepath.Join(dir, f.Name())) |
| if err != nil { |
| return nil, fmt.Errorf("failed to read file: %w", err) |
| } |
| var tests []test |
| if err := json.Unmarshal(b, &tests); err != nil { |
| return nil, fmt.Errorf("failed to unmarshal file %s: %w", f.Name(), err) |
| } |
| for _, test := range tests { |
| // Don't bother deserializing the default test data, we'll recompute |
| // it from scratch based on up-to-date data. |
| if test.Name == defaultTestName { |
| continue |
| } |
| if test.LastSeen.IsZero() { |
| test.LastSeen = date |
| } |
| res[builderName] = append(res[builderName], test) |
| } |
| } |
| return res, nil |
| } |
| |
| // splitTestsByBuilder takes a collection of tests and arranges them into a |
| // mapping corresponding to the data that should be written to each per-builder |
| // test duration file. |
| func splitTestsByBuilder(tests []test) (testDurationMap, error) { |
| // Mapping from builder name to list of tests. |
| durations := make(testDurationMap) |
| |
| for _, t := range tests { |
| if t.Builder == "" { |
| return nil, fmt.Errorf("test has empty builder: %+v", t) |
| } |
| durations[t.Builder] = append(durations[t.Builder], t) |
| } |
| |
| return durations, nil |
| } |
| |
| // calculateDefaultDurations aggregates durations for tests across all builders. |
| // testsharder will use this default file for any unknown builder that doesn't |
| // yet have its own durations file. |
| // |
| // We use the average duration of all tests, rather than the median, so that |
| // when using this default file, the sum of all tests' expected durations is |
| // close to the sum of actual durations. If we used the median of all durations, |
| // the sum of expected durations would likely be far too low because it wouldn't |
| // account for the long tail of slower tests, and we would end up producing too |
| // few shards when executing testsharder with a target per-shard duration. |
| // |
| // See unit tests for examples. |
| func calculateDefaultDurations(durations testDurationMap) ([]test, error) { |
| testsByName := make(map[string][]test) |
| for _, tests := range durations { |
| for _, t := range tests { |
| testsByName[t.Name] = append(testsByName[t.Name], t) |
| } |
| } |
| |
| var defaultDurations []test |
| for name, sameNameTests := range testsByName { |
| var totalRuns int64 |
| for _, t := range sameNameTests { |
| totalRuns += t.Runs |
| } |
| |
| duration, err := averageDurationMS(sameNameTests) |
| if err != nil { |
| return nil, err |
| } |
| |
| defaultDurations = append(defaultDurations, test{ |
| Name: name, |
| Builder: defaultBuilderName, |
| Runs: totalRuns, |
| MedianDurationMS: duration, |
| }) |
| } |
| |
| // The SQL query ensures that all other builders' tests are sorted, so we |
| // sort these too for consistency. |
| sort.Slice(defaultDurations, func(i, j int) bool { |
| return defaultDurations[i].Name < defaultDurations[j].Name |
| }) |
| |
| return defaultDurations, nil |
| } |
| |
| // addDefaultEntries adds an new entry for each builder at index zero that |
| // will be applied by testsharder to any test that doesn't have its own entry, |
| // probably because it's a new test or has been renamed. |
| // |
| // The average of all existing tests' median durations is probably a good |
| // estimate of the duration for any such tests. Average of medians is better |
| // than median of medians in the case where many new tests are added, assuming |
| // that the distribution of durations of the new tests is similar to the |
| // distribution for existing tests. (If only a couple tests are added, it |
| // doesn't make much of a difference either way). The median of medians might be |
| // closer to the actual duration of *most* of the new tests, but it doesn't take |
| // into account the fact that a few of the new tests are probably much longer |
| // than the rest. This would lead to all the new tests being put into the same |
| // shard, which would always time out because it has too many long tests. |
| func addDefaultEntries(durations testDurationMap) error { |
| for builder, builderTests := range durations { |
| defaultDuration, err := averageDurationMS(builderTests) |
| if err != nil { |
| return err |
| } |
| defaultTestEntry := test{ |
| Name: defaultTestName, |
| Builder: builder, |
| MedianDurationMS: defaultDuration, |
| } |
| durations[builder] = append([]test{defaultTestEntry}, builderTests...) |
| } |
| return nil |
| } |
| |
| // averageDurationMS calculates the average of median durations, weighted by |
| // runs, for all the given tests. |
| func averageDurationMS(tests []test) (int64, error) { |
| var totalDurationMS, totalRuns int64 |
| for _, t := range tests { |
| totalRuns += t.Runs |
| totalDurationMS += t.MedianDurationMS * t.Runs |
| } |
| if totalRuns == 0 { |
| // This likely indicates a bug somewhere in the SQL query or Go code. |
| return 0, fmt.Errorf("%w: %+v", errZeroTotalRuns, tests) |
| } |
| avg := float64(totalDurationMS) / float64(totalRuns) |
| return int64(math.Round(avg)), nil |
| } |
| |
| func writeDurations(durations testDurationMap, dir string, indentOutput bool) error { |
| for builder, tests := range durations { |
| functools.SortBy(tests, func(t test) string { |
| return t.Name |
| }) |
| var b []byte |
| var err error |
| if indentOutput { |
| b, err = json.MarshalIndent(tests, "", " ") |
| } else { |
| b, err = json.Marshal(tests) |
| } |
| if err != nil { |
| return err |
| } |
| |
| if indentOutput { |
| b = append(b, '\n') |
| } |
| fileName := fmt.Sprintf("%s.json", builder) |
| if err := os.WriteFile(filepath.Join(dir, fileName), b, 0o600); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |