catapult/histogram.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file

package catapult

import (
	"fmt"
	"log"
	"math"

	schema "fuchsia.googlesource.com/testing/perf/schema/v1"
	uuid "github.com/satori/go.uuid"
	"gonum.org/v1/gonum/stat"
)

// Histogram is a Catapult histogram object.
//
// See https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md
// for more information on the format.
//
// TODO(kjharland): Add these missing fields as needed
//   ShortName
//   BinBoundaries
//   NanDiagnostics
//   AllBins
//   SummaryOptions
type Histogram struct {
	Name               string    `json:"name"`
	GUID               string    `json:"guid"`
	Unit               string    `json:"unit"`
	Description        string    `json:"description"`
	SampleValues       []float64 `json:"sampleValues"`
	MaxNumSampleValues int       `json:"maxNumSampleValues"`
	NumNans            int       `json:"numNans"`
	Running            []float64 `json:"running"`
	// Diagnostics maps a Diagnostic's name to its GUID.
	//
	// These map entries communicate that the diagnostic with the given
	// name and GUID contains metadata that can help debug regressions and
	// other issues with this Histogram in the Catapult Dashboard.
	Diagnostics map[string]string `json:"diagnostics"`
}

// AddDiagnostic associates name with the given GUID in this Histogram's
// Diagnostics map.
//
// If the the new entry overwrites an existing entry, a warning is logged.
func (h *Histogram) AddDiagnostic(name string, guid string) {

	if h.Diagnostics == nil {
		h.Diagnostics = make(map[string]string)
	}

	if existing, ok := h.Diagnostics[name]; ok && existing != guid {
		log.Printf(
			"Overwriting shared Diagnostic %v in Histogram %v."+
				"($old, $new) = (%v, %v)",
			name, h.Name, existing, guid)
	}

	h.Diagnostics[name] = guid
}

// ConvertVariantsToHistograms converts a collection of Fuchsia benchmark
// variants into a list of Catapult Histograms.
func ConvertVariantsToHistograms(variants []schema.Variant) HistogramSet {
	histogramSet := HistogramSet{}
	for _, variant := range variants {
		for _, benchmarkData := range variant.FBenchmarksData {
			histogramSet.AddHistogram(
				createHistogram(variant, benchmarkData))
		}
	}

	return histogramSet
}

// Creates a histogram from the given variant and benchmark data set.
//
// TODO(kjharland): Generalize to suppport non-zircon benchmarks once I have
// a better idea of how other benchmarks will be converted.
func createHistogram(v schema.Variant, d schema.BenchmarkData) Histogram {
	var sampleValues []float64
	for _, sample := range d.Samples {
		// All zircon benchmarks use nanoseconds. Catapult doesn't support this,
		// so convert to milliseconds instead.
		for _, nanoseconds := range sample.Values {
			sampleValues = append(sampleValues, nanoseconds/1e6)
		}
	}

	return Histogram{
		Name:         fmt.Sprintf("%v, %v", v.VariantDesc, d.Label),
		Unit:         "ms_smallerIsBetter",
		GUID:         uuid.NewV4().String(),
		NumNans:      0, // All samples are numeric values
		SampleValues: sampleValues,
		// FIXME(kjharland): Define a static value for maxNumSampleValues, but what?
		MaxNumSampleValues: len(sampleValues),
		Running:            computeRunningStatistics(sampleValues),
	}
}

// Computes an ordered set of 7 statistics for the given set of values:
//
// count, max, meanlogs, mean, min, sum, variance
//
// https://github.com/catapult-project/catapult/issues/4150
func computeRunningStatistics(values []float64) []float64 {
	count := float64(len(values))
	min := math.Inf(1)
	max := math.Inf(-1)
	var sum float64
	for _, v := range values {
		min = math.Min(min, v)
		max = math.Max(max, v)
		sum += v
	}

	mean := stat.Mean(values, nil)
	variance := stat.Variance(values, nil)
	var meanlogs float64 // FIXME(kjharland): figure out what 'meanlogs' is.
	return []float64{count, max, meanlogs, mean, min, sum, variance}
}