catapult/histogram.go

// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file

package catapult

import (
	"fmt"
	"math"

	schema "fuchsia.googlesource.com/testing/perf/schema/v1"
	uuid "github.com/satori/go.uuid"
	"gonum.org/v1/gonum/stat"
)

// Histogram is a Catapult histogram object.
//
// See https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md
// for more information on the format.
//
// TODO(kjharland): Add these missing fields as needed
//   ShortName
//   BinBoundaries
//   NanDiagnostics
//   AllBins
//   SummaryOptions
type Histogram struct {
	Name               string            `json:"name"`
	GUID               string            `json:"guid"`
	Unit               string            `json:"unit"`
	Description        string            `json:"description"`
	Diagnostics        map[string]string `json:"diagnostics"`
	SampleValues       []float64         `json:"sampleValues"`
	MaxNumSampleValues int               `json:"maxNumSampleValues"`
	NumNans            int               `json:"numNans"`
	Running            []float64         `json:"running"`
}

// ConvertVariantsToHistograms converts a collection of Fuchsia benchmark variants into
// a list of Catapult Histograms.
func ConvertVariantsToHistograms(variants []schema.Variant) []Histogram {
	var histogramSet []Histogram
	for _, variant := range variants {
		for _, benchmarkData := range variant.FBenchmarksData {
			histogramSet = append(
				histogramSet,
				createHistogram(variant, benchmarkData),
			)
		}
	}

	return histogramSet
}

// Creates a histogram from the given variant and benchmark data set.
//
// TODO(kjharland): Generalize to suppport non-zircon benchmarks once I have
// a better idea of how other benchmarks will be converted.
func createHistogram(v schema.Variant, d schema.BenchmarkData) Histogram {
	var sampleValues []float64
	for _, sample := range d.Samples {
		// All zircon benchmarks use nanoseconds. Catapult doesn't support this,
		// so convert to milliseconds instead.
		for _, nanoseconds := range sample.Values {
			sampleValues = append(sampleValues, nanoseconds/1e6)
		}
	}

	return Histogram{
		Name:         fmt.Sprintf("%v, %v", v.VariantDesc, d.Label),
		Unit:         "ms_smallerIsBetter",
		GUID:         uuid.NewV4().String(),
		NumNans:      0, // All samples are numeric values
		SampleValues: sampleValues,
		// FIXME(kjharland): Define a static value for maxNumSampleValues, but what?
		MaxNumSampleValues: len(sampleValues),
		Running:            computeRunningStatistics(sampleValues),
	}
}

// Computes an ordered set of 7 statistics for the given set of values:
//
// count, max, meanlogs, mean, min, sum, variance
//
// https://github.com/catapult-project/catapult/issues/4150
func computeRunningStatistics(values []float64) []float64 {
	count := float64(len(values))
	min := math.Inf(1)
	max := math.Inf(-1)
	var sum float64
	for _, v := range values {
		min = math.Min(min, v)
		max = math.Max(max, v)
		sum += v
	}

	mean := stat.Mean(values, nil)
	variance := stat.Variance(values, nil)
	var meanlogs float64 // FIXME(kjharland): figure out what 'meanlogs' is.
	return []float64{count, max, meanlogs, mean, min, sum, variance}
}