blob: d52b21b66675c20e4555d3814ef7cfa9202fe781 [file] [log] [blame]
// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"fuchsia.googlesource.com/testing/catapult"
schema "fuchsia.googlesource.com/testing/perf/schema/v1"
"github.com/google/subcommands"
uuid "github.com/satori/go.uuid"
)
// sharedDiagnostics contains the values to use in the HistogramSet generated
// by this subcommand.
//
// Catapult requires a specific set of shared diagnostics to be present on a
// HistogramSet before uploading to the dashboard. Those required diagnostics
// are:
//
// * bots - a GenericSet of strings containing bot hostnames
// * benchmarks - a GenericSet of strings containing Telemetry
// benchmark names.
// * chromiumCommitPositions - a GenericSet of numbers containing
// Chromium commit positions.
// * masters - a GenericSet of strings containing buildbot master hostnames.
//
// There are other, optional diagnostics. Those may be added to this struct as
// needed.
//
// Because the names of some diagnostics are not relevant to users of this tool
// it is ok for this subcommand and this struct to use different names for any
// of them so long as the names are mapped back to their counterparts when the
// HistogramSet is constructed.
//
// To learn more about Catapult Diagnostics, see:
// https://github.com/catapult-project/catapult/blob/master/docs/how-to-write-metrics.md
type sharedDiagnostics struct {
// testSuite specifies the test suite that we are creating a
// HistogramSet for. e.g. "zircon" "ledger"
//
// Corresponds to the diagnostic "benchmarks".
testSuite string
// builder specifies the LUCI builder that ran these tests.
//
// Corresponds to the diagnostic "bots".
builder string
// bucket specifies the LUCI bucket containing the job that ran
// these tests.
//
// Corresponds to the diagnostic "masters".
bucket string
// dateTime marks the time when tests were executed in ms since the UNIX
// Epoch. The Catapult dashboard uses this value to order sample points in
// a graph.
//
// Corresponds to the diagnostic "chromiumCommitPositions".
dateTime uint64
}
type MakeHistogramCommand struct {
sharedDiagnostics sharedDiagnostics
}
func (*MakeHistogramCommand) Name() string {
return "make_histogram"
}
func (*MakeHistogramCommand) Usage() string {
return "make_histogram [options] input_file"
}
func (*MakeHistogramCommand) Synopsis() string {
return "Converts performance test output to a catapult HistogramSet"
}
func (cmd *MakeHistogramCommand) SetFlags(flags *flag.FlagSet) {
flags.StringVar(&cmd.sharedDiagnostics.testSuite, "test-suite", "",
"Test suite corresponding to the input file")
flags.StringVar(&cmd.sharedDiagnostics.builder, "builder", "",
"LUCI builder that generated the test results")
flags.StringVar(&cmd.sharedDiagnostics.bucket, "bucket", "",
"Buildbucket bucket containing the job that ran the test")
flags.Uint64Var(&cmd.sharedDiagnostics.dateTime,
"datetime", 0,
"The date (in ms since epoch) the given tests were executed")
}
// Execute converts performance test output to a catapult HistogramSet.
//
// A shared diagnostic will appear in the resulting HistogramSet for each
// diagnostic flag specified when invoking this subcommand.
//
// For more information about the HistogramSet format, See
// https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md
func (cmd *MakeHistogramCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
if f.NArg() != 1 {
log.Println("missing input_file")
return subcommands.ExitFailure
}
inputFile := f.Arg(0)
blob, err := ioutil.ReadFile(inputFile)
if err != nil {
log.Println(err)
return subcommands.ExitFailure
}
histogramSet := new(catapult.HistogramSet)
var variants struct {
Variants []schema.Variant `json:"variants"`
}
if err := json.Unmarshal(blob, &variants); err != nil {
log.Println(err)
return subcommands.ExitFailure
}
// Generate Histograms from test results.
histograms := catapult.ConvertVariantsToHistograms(variants.Variants)
// Track whether any single Histogram addition failed. We do this instead of
// exiting after a single error to expose as many errors as possible.
anyAddFailed := false
for _, histogram := range histograms {
if err := histogramSet.AddHistogram(histogram); err != nil {
log.Println(err)
anyAddFailed = true
}
}
if anyAddFailed {
return subcommands.ExitFailure
}
// Add all shared diagnostics, mapping to the appropriate Catapult
// diagnostic names.
diagnosticInfoPairs := []struct {
Name string
Value interface{}
}{
// chromiumCommitPositions must come first or Catapult will reject the upload.
{"chromiumCommitPositions", cmd.sharedDiagnostics.dateTime},
{"benchmarks", cmd.sharedDiagnostics.testSuite},
{"bots", cmd.sharedDiagnostics.builder},
{"masters", cmd.sharedDiagnostics.bucket},
}
for _, info := range diagnosticInfoPairs {
histogramSet.AddSharedDiagnostic(info.Name, &catapult.GenericSetDiagnostic{
Type: catapult.DiagnosticTypeGenericSet,
GUID: uuid.NewV4().String(),
Values: []interface{}{info.Value},
})
}
// Serialize and log the new HistogramSet.
js, err := json.Marshal(histogramSet.ToJSON())
if err != nil {
log.Println(err)
return subcommands.ExitFailure
}
// TODO(kjharland): Add option to write to a file so this can be used
// from a recipe.
fmt.Println(string(js))
return subcommands.ExitSuccess
}