| // Copyright 2017 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| package main |
| |
| import ( |
| "context" |
| "encoding/json" |
| "flag" |
| "fmt" |
| "io/ioutil" |
| "log" |
| |
| "fuchsia.googlesource.com/testing/catapult" |
| schema "fuchsia.googlesource.com/testing/perf/schema/v1" |
| "github.com/google/subcommands" |
| uuid "github.com/satori/go.uuid" |
| ) |
| |
| // sharedDiagnostics contains the values to use in the HistogramSet generated |
| // by this subcommand. |
| // |
| // Catapult requires a specific set of shared diagnostics to be present on a |
| // HistogramSet before uploading to the dashboard. Those required diagnostics |
| // are: |
| // |
| // * bots - a GenericSet of strings containing bot hostnames |
| // * benchmarks - a GenericSet of strings containing Telemetry |
| // benchmark names. |
| // * chromiumCommitPositions - a GenericSet of numbers containing |
| // Chromium commit positions. |
| // * masters - a GenericSet of strings containing buildbot master hostnames. |
| // |
| // There are other, optional diagnostics. Those may be added to this struct as |
| // needed. |
| // |
| // Because the names of some diagnostics are not relevant to users of this tool |
| // it is ok for this subcommand and this struct to use different names for any |
| // of them so long as the names are mapped back to their counterparts when the |
| // HistogramSet is constructed. |
| // |
| // To learn more about Catapult Diagnostics, see: |
| // https://github.com/catapult-project/catapult/blob/master/docs/how-to-write-metrics.md |
| type sharedDiagnostics struct { |
| // testSuite specifies the test suite that we are creating a |
| // HistogramSet for. e.g. "zircon" "ledger" |
| // |
| // Corresponds to the diagnostic "benchmarks". |
| testSuite string |
| |
| // builder specifies the LUCI builder that ran these tests. |
| // |
| // Corresponds to the diagnostic "bots". |
| builder string |
| |
| // bucket specifies the LUCI bucket containing the job that ran |
| // these tests. |
| // |
| // Corresponds to the diagnostic "masters". |
| bucket string |
| |
| // dateTime marks the time when tests were executed in ms since the UNIX |
| // Epoch. The Catapult dashboard uses this value to order sample points in |
| // a graph. |
| // |
| // Corresponds to the diagnostic "chromiumCommitPositions". |
| dateTime uint64 |
| } |
| |
| type MakeHistogramCommand struct { |
| sharedDiagnostics sharedDiagnostics |
| } |
| |
| func (*MakeHistogramCommand) Name() string { |
| return "make_histogram" |
| } |
| |
| func (*MakeHistogramCommand) Usage() string { |
| return "make_histogram [options] input_file" |
| } |
| |
| func (*MakeHistogramCommand) Synopsis() string { |
| return "Converts performance test output to a catapult HistogramSet" |
| } |
| |
| func (cmd *MakeHistogramCommand) SetFlags(flags *flag.FlagSet) { |
| flags.StringVar(&cmd.sharedDiagnostics.testSuite, "test-suite", "", |
| "Test suite corresponding to the input file") |
| flags.StringVar(&cmd.sharedDiagnostics.builder, "builder", "", |
| "LUCI builder that generated the test results") |
| flags.StringVar(&cmd.sharedDiagnostics.bucket, "bucket", "", |
| "Buildbucket bucket containing the job that ran the test") |
| flags.Uint64Var(&cmd.sharedDiagnostics.dateTime, |
| "datetime", 0, |
| "The date (in ms since epoch) the given tests were executed") |
| } |
| |
| // Execute converts performance test output to a catapult HistogramSet. |
| // |
| // A shared diagnostic will appear in the resulting HistogramSet for each |
| // diagnostic flag specified when invoking this subcommand. |
| // |
| // For more information about the HistogramSet format, See |
| // https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md |
| func (cmd *MakeHistogramCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { |
| if f.NArg() != 1 { |
| log.Println("missing input_file") |
| return subcommands.ExitFailure |
| } |
| |
| inputFile := f.Arg(0) |
| blob, err := ioutil.ReadFile(inputFile) |
| if err != nil { |
| log.Println(err) |
| return subcommands.ExitFailure |
| } |
| |
| histogramSet := new(catapult.HistogramSet) |
| |
| var variants struct { |
| Variants []schema.Variant `json:"variants"` |
| } |
| |
| if err := json.Unmarshal(blob, &variants); err != nil { |
| log.Println(err) |
| return subcommands.ExitFailure |
| } |
| |
| // Generate Histograms from test results. |
| histograms := catapult.ConvertVariantsToHistograms(variants.Variants) |
| |
| // Track whether any single Histogram addition failed. We do this instead of |
| // exiting after a single error to expose as many errors as possible. |
| anyAddFailed := false |
| for _, histogram := range histograms { |
| if err := histogramSet.AddHistogram(histogram); err != nil { |
| log.Println(err) |
| anyAddFailed = true |
| } |
| } |
| if anyAddFailed { |
| return subcommands.ExitFailure |
| } |
| |
| // Add all shared diagnostics, mapping to the appropriate Catapult |
| // diagnostic names. |
| diagnosticInfoPairs := []struct { |
| Name string |
| Value interface{} |
| }{ |
| // chromiumCommitPositions must come first or Catapult will reject the upload. |
| {"chromiumCommitPositions", cmd.sharedDiagnostics.dateTime}, |
| {"benchmarks", cmd.sharedDiagnostics.testSuite}, |
| {"bots", cmd.sharedDiagnostics.builder}, |
| {"masters", cmd.sharedDiagnostics.bucket}, |
| } |
| for _, info := range diagnosticInfoPairs { |
| histogramSet.AddSharedDiagnostic(info.Name, &catapult.GenericSetDiagnostic{ |
| Type: catapult.DiagnosticTypeGenericSet, |
| GUID: uuid.NewV4().String(), |
| Values: []interface{}{info.Value}, |
| }) |
| } |
| |
| // Serialize and log the new HistogramSet. |
| js, err := json.Marshal(histogramSet.ToJSON()) |
| if err != nil { |
| log.Println(err) |
| return subcommands.ExitFailure |
| } |
| |
| // TODO(kjharland): Add option to write to a file so this can be used |
| // from a recipe. |
| fmt.Println(string(js)) |
| return subcommands.ExitSuccess |
| } |