| // Copyright 2018 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file |
| |
| package catapult |
| |
| import ( |
| "fmt" |
| "log" |
| "math" |
| |
| schema "fuchsia.googlesource.com/testing/perf/schema/v1" |
| uuid "github.com/satori/go.uuid" |
| "gonum.org/v1/gonum/stat" |
| ) |
| |
| // Histogram is a Catapult histogram object. |
| // |
| // See https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md |
| // for more information on the format. |
| // |
| // TODO(kjharland): Add these missing fields as needed |
| // ShortName |
| // BinBoundaries |
| // NanDiagnostics |
| // AllBins |
| // SummaryOptions |
| type Histogram struct { |
| Name string `json:"name"` |
| GUID string `json:"guid"` |
| Unit string `json:"unit"` |
| Description string `json:"description"` |
| SampleValues []float64 `json:"sampleValues"` |
| MaxNumSampleValues int `json:"maxNumSampleValues"` |
| NumNans int `json:"numNans"` |
| Running []float64 `json:"running"` |
| // Diagnostics maps a Diagnostic's name to its GUID. |
| // |
| // These map entries communicate that the diagnostic with the given |
| // name and GUID contains metadata that can help debug regressions and |
| // other issues with this Histogram in the Catapult Dashboard. |
| Diagnostics map[string]string `json:"diagnostics"` |
| } |
| |
| // AddDiagnostic associates name with the given GUID in this Histogram's |
| // Diagnostics map. |
| // |
| // If the the new entry overwrites an existing entry, a warning is logged. |
| func (h *Histogram) AddDiagnostic(name string, guid string) { |
| |
| if h.Diagnostics == nil { |
| h.Diagnostics = make(map[string]string) |
| } |
| |
| if existing, ok := h.Diagnostics[name]; ok && existing != guid { |
| log.Printf( |
| "Overwriting shared Diagnostic %v in Histogram %v."+ |
| "($old, $new) = (%v, %v)", |
| name, h.Name, existing, guid) |
| } |
| |
| h.Diagnostics[name] = guid |
| } |
| |
| // ConvertVariantsToHistograms converts a collection of Fuchsia benchmark |
| // variants into a list of Catapult Histograms. |
| func ConvertVariantsToHistograms(variants []schema.Variant) []Histogram { |
| var histograms []Histogram |
| for _, variant := range variants { |
| for _, benchmarkData := range variant.FBenchmarksData { |
| histograms = append(histograms, |
| createHistogram(variant, benchmarkData)) |
| } |
| } |
| |
| return histograms |
| } |
| |
| // Creates a histogram from the given variant and benchmark data set. |
| // |
| // TODO(kjharland): Generalize to suppport non-zircon benchmarks once I have |
| // a better idea of how other benchmarks will be converted. |
| func createHistogram(v schema.Variant, d schema.BenchmarkData) Histogram { |
| var sampleValues []float64 |
| for _, sample := range d.Samples { |
| // All zircon benchmarks use nanoseconds. Catapult doesn't support this, |
| // so convert to milliseconds instead. |
| for _, nanoseconds := range sample.Values { |
| sampleValues = append(sampleValues, nanoseconds/1e6) |
| } |
| } |
| |
| return Histogram{ |
| Name: fmt.Sprintf("%v, %v", v.VariantDesc, d.Label), |
| Unit: "ms_smallerIsBetter", |
| GUID: uuid.NewV4().String(), |
| NumNans: 0, // All samples are numeric values |
| SampleValues: sampleValues, |
| // FIXME(kjharland): Define a static value for maxNumSampleValues, but what? |
| MaxNumSampleValues: len(sampleValues), |
| Running: computeRunningStatistics(sampleValues), |
| } |
| } |
| |
| // Computes an ordered set of 7 statistics for the given set of values: |
| // |
| // count, max, meanlogs, mean, min, sum, variance |
| // |
| // https://github.com/catapult-project/catapult/issues/4150 |
| func computeRunningStatistics(values []float64) []float64 { |
| count := float64(len(values)) |
| min := math.Inf(1) |
| max := math.Inf(-1) |
| var sum float64 |
| for _, v := range values { |
| min = math.Min(min, v) |
| max = math.Max(max, v) |
| sum += v |
| } |
| |
| mean := stat.Mean(values, nil) |
| variance := stat.Variance(values, nil) |
| var meanlogs float64 // FIXME(kjharland): figure out what 'meanlogs' is. |
| return []float64{count, max, meanlogs, mean, min, sum, variance} |
| } |