Deprecate obsolete repository Change-Id: I4f898cba2635a54f125740d375466b1ff8d44cea Reviewed-on: https://fuchsia-review.googlesource.com/c/testing/+/418719 Reviewed-by: Mahesh Saripalli <maheshsr@google.com> Commit-Queue: Mahesh Saripalli <maheshsr@google.com>
diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 4857869..0000000 --- a/.gitignore +++ /dev/null
@@ -1,89 +0,0 @@ -# swap -[._]*.s[a-w][a-z] -[._]s[a-w][a-z] -# session -Session.vim -# temporary -.netrwhist -*~ -# auto-generated tag files -tags -# -*- mode: gitignore; -*- -*~ -\#*\# -/.emacs.desktop -/.emacs.desktop.lock -*.elc -auto-save-list -tramp -.\#* - -# Org-mode -.org-id-locations -*_archive - -# flymake-mode -*_flymake.* - -# eshell files -/eshell/history -/eshell/lastdir - -# elpa packages -/elpa/ - -# reftex files -*.rel - -# AUCTeX auto folder -/auto/ - -# cask packages -.cask/ -dist/ - -# Flycheck -flycheck_*.el - -# server auth directory -/server/ - -# projectiles files -.projectile*~ - -# temporary files which can be created if a process still has a handle open of a deleted file -.fuse_hidden* - -# KDE directory preferences -.directory - -# Linux trash folder which might appear on any partition or disk -.Trash-* -*.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -vendor/ \ No newline at end of file
diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index d09a369..0000000 --- a/AUTHORS +++ /dev/null
@@ -1,7 +0,0 @@ -# This is the list of Fuchsia Authors. - -# Names should be added to this file as one of -# Organization's name -# Individual's name <submission email address> - -Google Inc.
diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 9fd2067..0000000 --- a/Gopkg.lock +++ /dev/null
@@ -1,65 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/davecgh/go-spew" - packages = ["spew"] - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" - -[[projects]] - branch = "master" - name = "github.com/google/subcommands" - packages = ["."] - revision = "ce3d4cfc062faac7115d44e5befec8b5a08c3faa" - -[[projects]] - name = "github.com/pmezard/go-difflib" - packages = ["difflib"] - revision = "792786c7400a136282c1664665ae0a8db921c6c2" - version = "v1.0.0" - -[[projects]] - name = "github.com/satori/go.uuid" - packages = ["."] - revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" - source = "https://github.googlesource.com/satori/go.uuid.git" - version = "v1.2.0" - -[[projects]] - name = "github.com/stretchr/testify" - packages = [ - "assert", - "require", - "suite" - ] - revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c" - version = "v1.2.0" - -[[projects]] - branch = "master" - name = "gonum.org/v1/gonum" - packages = [ - "blas", - "blas/blas64", - "blas/gonum", - "floats", - "internal/asm/c128", - "internal/asm/f32", - "internal/asm/f64", - "internal/math32", - "lapack", - "lapack/gonum", - "lapack/lapack64", - "mat", - "stat" - ] - revision = "69fc04c7c31754cf196d3bcc70c3bc0eec9da1b7" - source = "https://github.googlesource.com/gonum/gonum.git" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "44434b6fb39430fb1d23a531e44c2204bbcfb0cc1d0fc4377678c8d044c61374" - solver-name = "gps-cdcl" - solver-version = 1
diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index 3636368..0000000 --- a/Gopkg.toml +++ /dev/null
@@ -1,9 +0,0 @@ -[[constraint]] - name = "github.com/satori/go.uuid" - version = "1.1.0" - source = "https://github.googlesource.com/satori/go.uuid.git" - -[[constraint]] - branch = "master" - name = "gonum.org/v1/gonum" - source = "https://github.googlesource.com/gonum/gonum.git" \ No newline at end of file
diff --git a/LICENSE b/LICENSE deleted file mode 100644 index c18a59e..0000000 --- a/LICENSE +++ /dev/null
@@ -1,27 +0,0 @@ -Copyright 2017 The Fuchsia Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/PATENTS b/PATENTS deleted file mode 100644 index 2746e78..0000000 --- a/PATENTS +++ /dev/null
@@ -1,22 +0,0 @@ -Additional IP Rights Grant (Patents) - -"This implementation" means the copyrightable works distributed by -Google as part of the Fuchsia project. - -Google hereby grants to you a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this -section) patent license to make, have made, use, offer to sell, sell, -import, transfer, and otherwise run, modify and propagate the contents -of this implementation of Fuchsia, where such license applies only to -those patent claims, both currently owned by Google and acquired in -the future, licensable by Google that are necessarily infringed by -this implementation. This grant does not include claims that would be -infringed only as a consequence of further modification of this -implementation. If you or your agent or exclusive licensee institute -or order or agree to the institution of patent litigation or any other -patent enforcement activity against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that this -implementation of Fuchsia constitutes direct or contributory patent -infringement, or inducement of patent infringement, then any patent -rights granted to you under this License for this implementation of -Fuchsia shall terminate as of the date such litigation is filed.
diff --git a/README.md b/README.md index 72a662b..3cc8b3b 100644 --- a/README.md +++ b/README.md
@@ -1,3 +1,5 @@ -# Testing +# Obsolete -This repository is for scripts and tools useful when testing on Fuchsia. +This repository has moved into Fuchsia's main repository: +https://fuchsia.googlesource.com/fuchsia/ +
diff --git a/catapult/.gitignore b/catapult/.gitignore deleted file mode 100644 index a1338d6..0000000 --- a/catapult/.gitignore +++ /dev/null
@@ -1,14 +0,0 @@ -# Binaries for programs and plugins -*.exe -*.dll -*.so -*.dylib - -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 -.glide/
diff --git a/catapult/README.md b/catapult/README.md deleted file mode 100644 index 7fd1796..0000000 --- a/catapult/README.md +++ /dev/null
@@ -1,18 +0,0 @@ -Catapult -======== - -Go packages for working with Catapult. - -#### Example Usage: -Convert Fuchsia performance test output into a HistogramSet for the Catapult dashboard: - -```sh -catapult make_histogram --datetime=1234455667789 --test-suite=fuchsia.example.benchmark --builder=fuchsia.example.builder --bucket=fuchsia.example.bucket /path/to/test_results.json > histogram_set.json -``` - -Upload a file containing a HistogramSet to the Catapult dashboard: - -```sh -catapult upload --url=https://chromeperf.appspot.com/add_histograms --service-account-json=/path/to/acccount.json histogram_set.json -``` -
diff --git a/catapult/diagnostic.go b/catapult/diagnostic.go deleted file mode 100644 index b819ea0..0000000 --- a/catapult/diagnostic.go +++ /dev/null
@@ -1,31 +0,0 @@ -// Copyright 2018 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file - -package catapult - -// Types of Catapult Diagnostics. -// -// Every Diagnostic must have a Type value from this list. For a list of all -// Diagnostic types, see: -// https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md#diagnostics -const ( - DiagnosticTypeGenericSet = "GenericSet" -) - -// Diagnostic is an interface for Catapult Diagnostics. -type Diagnostic interface { - // GetGUID returns the GUID of this diagnostic. - GetGUID() string -} - -// GenericSetDiagnostic stores arbitary untyped data in Histograms. -type GenericSetDiagnostic struct { - Type string `json:"type"` - GUID string `json:"guid"` - Values []interface{} `json:"values"` -} - -func (d *GenericSetDiagnostic) GetGUID() string { - return d.GUID -}
diff --git a/catapult/histogram.go b/catapult/histogram.go deleted file mode 100644 index fcb6fff..0000000 --- a/catapult/histogram.go +++ /dev/null
@@ -1,140 +0,0 @@ -// Copyright 2018 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file - -package catapult - -import ( - "fmt" - "log" - "math" - - schema "fuchsia.googlesource.com/testing/perf/schema/v1" - uuid "github.com/satori/go.uuid" - "gonum.org/v1/gonum/stat" -) - -// Histogram is a Catapult histogram object. -// -// See https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md -// for more information on the format. -// -// TODO(kjharland): Add these missing fields as needed -// ShortName -// BinBoundaries -// NanDiagnostics -// AllBins -// SummaryOptions -type Histogram struct { - Name string `json:"name"` - GUID string `json:"guid"` - Unit string `json:"unit"` - Description string `json:"description"` - SampleValues []float64 `json:"sampleValues"` - MaxNumSampleValues int `json:"maxNumSampleValues"` - NumNans int `json:"numNans"` - Running []float64 `json:"running"` - // Diagnostics maps a Diagnostic's name to its GUID. - // - // These map entries communicate that the diagnostic with the given - // name and GUID contains metadata that can help debug regressions and - // other issues with this Histogram in the Catapult Dashboard. - Diagnostics map[string]string `json:"diagnostics"` -} - -// AddDiagnostic associates name with the given GUID in this Histogram's -// Diagnostics map. -// -// If the the new entry overwrites an existing entry, a warning is logged. -func (h *Histogram) AddDiagnostic(name string, guid string) { - - if h.Diagnostics == nil { - h.Diagnostics = make(map[string]string) - } - - if existing, ok := h.Diagnostics[name]; ok && existing != guid { - log.Printf( - "Overwriting shared Diagnostic %v in Histogram %v."+ - "($old, $new) = (%v, %v)", - name, h.Name, existing, guid) - } - - h.Diagnostics[name] = guid -} - -// ConvertVariantsToHistograms converts a collection of Fuchsia benchmark -// variants into a list of Catapult Histograms. -func ConvertVariantsToHistograms(variants []schema.Variant) []Histogram { - var histograms []Histogram - for _, variant := range variants { - for _, benchmarkData := range variant.FBenchmarksData { - histograms = append(histograms, - createHistogram(variant, benchmarkData)) - } - } - - return histograms -} - -// Creates a histogram from the given variant and benchmark data set. -// -// TODO(kjharland): Generalize to support non-zircon benchmarks once I have -// a better idea of how other benchmarks will be converted. -func createHistogram(v schema.Variant, d schema.BenchmarkData) Histogram { - var sampleValues []float64 - for _, sample := range d.Samples { - // The number of samples to take. Catapult will reject sample - // sets that are *too large*. Fuchsia tests usually generate - // ~10,000 - 20,000 sample points per test-run, but only a - // handful of these are needed. - // - // TODO(kjharland): Use a sane max of the first 1000 points for now. - // Find out how to properly split samples so that data is not thrown - // away. - numSamples := math.Min(1000, float64(len(sample.Values))) - - // All zircon benchmarks use nanoseconds. Catapult doesn't support this, - // so convert to milliseconds instead. - for i := 0; i < int(numSamples); i++ { - sampleValues = append(sampleValues, sample.Values[i]/1e6) - } - } - - return Histogram{ - Name: fmt.Sprintf("%v, %v", v.VariantDesc, d.Label), - Unit: "ms_smallerIsBetter", - GUID: uuid.NewV4().String(), - NumNans: 0, // All samples are numeric values - SampleValues: sampleValues, - MaxNumSampleValues: len(sampleValues), - Running: computeRunningStatistics(sampleValues), - } -} - -// Computes an ordered set of 7 statistics for the given set of values: -// -// count, max, meanlogs, mean, min, sum, variance -// -// meanlogs is the mean of the logs of the absolute values of the given values. -// -// https://github.com/catapult-project/catapult/issues/4150 -func computeRunningStatistics(values []float64) []float64 { - count := float64(len(values)) - min := math.Inf(1) - max := math.Inf(-1) - var sum float64 - var meanlogs float64 - - for i, v := range values { - min = math.Min(min, v) - max = math.Max(max, v) - sum += v - // Compute meanlogs as a cumulative moving average: - // https://en.wikipedia.org/wiki/Moving_average - meanlogs += (math.Log10(math.Abs(v)) - meanlogs) / float64(i+1) - } - - mean := stat.Mean(values, nil) - variance := stat.Variance(values, nil) - return []float64{count, max, meanlogs, mean, min, sum, variance} -}
diff --git a/catapult/histogram_set.go b/catapult/histogram_set.go deleted file mode 100644 index 71c40d1..0000000 --- a/catapult/histogram_set.go +++ /dev/null
@@ -1,67 +0,0 @@ -// Copyright 2018 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file - -package catapult - -import ( - "errors" -) - -// HistogramSet is a collection of Histograms and related Diagnostics. -type HistogramSet struct { - histograms []Histogram - diagnostics []Diagnostic -} - -// AddHistogram adds a histogram to this HistogramSet. -// -// If this HistogramSet already contains a Diagnostic, the addition fails and -// an error is returned. -// -// TODO(kharland): This API does not allow adding a Histogram after Diagnostics -// have been added because it is not clear if the caller intended for the new -// Histogram to contain the same references to the shared Diagnostics in this -// HistogramSet as the existing Histograms. Allow adding Histograms after -// Diagnostics when Fuchsia needs this functionality. -func (h *HistogramSet) AddHistogram(histogram Histogram) error { - if len(h.diagnostics) > 0 { - return errors.New("can't add Histogram after adding Diagnostics") - } - h.histograms = append(h.histograms, histogram) - return nil -} - -// AddSharedDiagnostic adds a diagnostic to this HistogramSet. -// -// A shared Diagnostic is a diagnostic that exists in the HistogramSet, so -// that it may be referenced by Histograms within the set. A reference to the -// Diagnostic is automatically added to each Histogram in the set when calling -// this method. name is used as the key for this reference. -func (h *HistogramSet) AddSharedDiagnostic(name string, diagnostic Diagnostic) { - h.diagnostics = append(h.diagnostics, diagnostic) - - guid := diagnostic.GetGUID() - - // Reference the diagnostic's GUID in each Histogram in this set. - // - // Use the loop index instead of the loop variable since this directly - // mutates the histogram. - for i := range h.histograms { - h.histograms[i].AddDiagnostic(name, guid) - } -} - -// ToJSON returns a JSON-serializable representation of this HistogramSet. -func (h *HistogramSet) ToJSON() []interface{} { - jsonObject := []interface{}{} - // Diagnostics must come first in the HistogramSet or Catapult will reject - // the upload request. - for _, entry := range h.diagnostics { - jsonObject = append(jsonObject, entry) - } - for _, entry := range h.histograms { - jsonObject = append(jsonObject, entry) - } - return jsonObject -}
diff --git a/catapult/histogram_set_test.go b/catapult/histogram_set_test.go deleted file mode 100644 index 89d37fb..0000000 --- a/catapult/histogram_set_test.go +++ /dev/null
@@ -1,136 +0,0 @@ -// Copyright 2018 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file -package catapult_test - -import ( - "reflect" - "testing" - - "fuchsia.googlesource.com/testing/catapult" - "github.com/kr/pretty" -) - -func TestAddHistogram(t *testing.T) { - t.Run("When the HistogramSet is empty", func(t *testing.T) { - histogramSet := catapult.HistogramSet{} - histogram := catapult.Histogram{Name: "test_histogram"} - - expectedJSON := []interface{}{histogram} - histogramSet.AddHistogram(histogram) - - if !reflect.DeepEqual(histogramSet.ToJSON(), expectedJSON) { - t.Errorf("Invalid HistogramSet. Got %+v. Expected %+v", - pretty.Sprint(histogramSet.ToJSON()), - pretty.Sprint(expectedJSON), - ) - } - }) - - t.Run("When the HistogramSet contains other Histograms", func(t *testing.T) { - histogramSet := catapult.HistogramSet{} - histogram1 := catapult.Histogram{Name: "test_histogram_1"} - histogram2 := catapult.Histogram{Name: "test_histogram_2"} - - expectedJSON := []interface{}{histogram1, histogram2} - - histogramSet.AddHistogram(histogram1) - histogramSet.AddHistogram(histogram2) - if !reflect.DeepEqual(histogramSet.ToJSON(), expectedJSON) { - t.Errorf("Invalid HistogramSet. Got %+v. Expected %+v", - pretty.Sprint(histogramSet.ToJSON()), - pretty.Sprint(expectedJSON), - ) - } - }) - - t.Run("When the HistogramSet contains other Diagnostics", func(t *testing.T) { - histogramSet := catapult.HistogramSet{} - histogram := catapult.Histogram{ - Name: "test_histogram", - Diagnostics: make(map[string]string), - } - diagnostic := &catapult.GenericSetDiagnostic{ - Type: catapult.DiagnosticTypeGenericSet, - Values: []interface{}{"test_value"}, - GUID: "test_guid", - } - - // Should return an error without modifying the HistogramSet - // because all Histograms must be added before any Diagnotics - // are added. - expectedJSON := []interface{}{diagnostic} - histogramSet.AddSharedDiagnostic("test_diagnostic", diagnostic) - if histogramSet.AddHistogram(histogram) == nil { - t.Errorf("expected an error when adding a Histogram after a Diagnostic") - } - - if !reflect.DeepEqual(histogramSet.ToJSON(), expectedJSON) { - t.Errorf("Invalid HistogramSet. Got %+v. Expected %+v", - pretty.Sprint(histogramSet.ToJSON()), - pretty.Sprint(expectedJSON), - ) - } - }) -} - -func TestAddSharedDiagnostic(t *testing.T) { - t.Run("When there are no Histograms in the set", func(t *testing.T) { - diagnostic := &catapult.GenericSetDiagnostic{ - Type: catapult.DiagnosticTypeGenericSet, - GUID: "test_guid", - Values: []interface{}{"test_value"}, - } - histogramSet := catapult.HistogramSet{} - histogramSet.AddSharedDiagnostic("test_diagnostic", diagnostic) - - expectedJSON := []interface{}{diagnostic} - if !reflect.DeepEqual(histogramSet.ToJSON(), expectedJSON) { - t.Errorf("Invalid HistogramSet. Got: %+v\nExpected: %+v", - pretty.Sprint(histogramSet.ToJSON()), - pretty.Sprint(expectedJSON)) - } - - }) - - t.Run("When there are existing Histograms in the set", func(t *testing.T) { - diagnostic := &catapult.GenericSetDiagnostic{ - Type: catapult.DiagnosticTypeGenericSet, - GUID: "test_guid", - Values: []interface{}{"test_value"}, - } - - histogramSet := catapult.HistogramSet{} - histogramSet.AddHistogram(catapult.Histogram{ - Name: "test_histogram_1", - Diagnostics: make(map[string]string), - }) - histogramSet.AddHistogram(catapult.Histogram{ - Name: "test_histogram_2", - // Intentionally leaving this Diagnostics map as nil to - // make sure the map still recieves a reference to the - // new Diagnostic. - }) - histogramSet.AddSharedDiagnostic("test_diagnostic", diagnostic) - - // Each histogram in the set should have a reference to the diagnostic in its - // Diagnostics map. - expectedJSON := []interface{}{ - diagnostic, - catapult.Histogram{ - Name: "test_histogram_1", - Diagnostics: map[string]string{"test_diagnostic": "test_guid"}, - }, - catapult.Histogram{ - Name: "test_histogram_2", - Diagnostics: map[string]string{"test_diagnostic": "test_guid"}, - }, - } - - if !reflect.DeepEqual(histogramSet.ToJSON(), expectedJSON) { - t.Errorf("Invalid HistogramSet. Got: %+v\nExpected: %+v", - pretty.Sprint(histogramSet.ToJSON()), - pretty.Sprint(expectedJSON)) - } - }) -}
diff --git a/catapult/histogram_test.go b/catapult/histogram_test.go deleted file mode 100644 index 524948b..0000000 --- a/catapult/histogram_test.go +++ /dev/null
@@ -1,131 +0,0 @@ -// Copyright 2018 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file -package catapult_test - -import ( - "encoding/json" - "reflect" - "testing" - - . "fuchsia.googlesource.com/testing/catapult" - schema "fuchsia.googlesource.com/testing/perf/schema/v1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" -) - -type HistogramTestSuite struct { - suite.Suite -} - -func TestHistogram(t *testing.T) { - suite.Run(t, new(HistogramTestSuite)) -} - -// TODO(kjharland): After writing this test the team decided we shouldn't use -// assertion libraries. Remove `assert.*` references in a later change. -func (s *HistogramTestSuite) TestConvertVariantsToHistograms() { - var perfTestOutput = []byte(`{ - "variants": [{ - "finish_time": 1234567, - "variant_desc": "example_variant", - "boot_args": "example_boot_arg1", - "fbenchmarks_data": [{ - "label": "example_benchmark_data", - "unit": "ns", - "samples": [{ - "label": "example_sample", - "values": [10, 20, 30, 40, 50] - }] - }] - }] - } - `) - - var variants struct { - Variants []schema.Variant - } - - json.Unmarshal(perfTestOutput, &variants) - histograms := ConvertVariantsToHistograms(variants.Variants) - - assert.Equal(s.T(), len(histograms), 1) - - histogram := histograms[0] - assert.Equal(s.T(), histogram.Name, "example_variant, example_benchmark_data") - assert.Len(s.T(), histogram.GUID, 36) - assert.Equal(s.T(), histogram.Unit, "ms_smallerIsBetter") - assert.Empty(s.T(), histogram.Description) - assert.Empty(s.T(), histogram.Diagnostics) - assert.ElementsMatch(s.T(), histogram.SampleValues, []float64{ - 1e-5, - 2e-5, - 3e-5, - 4e-5, - 5e-5, - }) - assert.Equal(s.T(), histogram.MaxNumSampleValues, 5) - assert.Equal(s.T(), histogram.NumNans, 0) - - assert.ElementsMatch(s.T(), histogram.Running, []float64{ - 5, // count - 5e-05, // max - -4.5841637507904744, // meanlogs - 3.0000000000000004e-05, //mean - 1e-5, // min - 1.5000000000000001e-4, //sum - 2.5e-10, // variance - }) -} - -func (s *HistogramTestSuite) TestHistogram_AddDiagnostic() { - var testName, testGUID string - var histogram Histogram - - setUp := func() { - testName = "test-name" - testGUID = "test-guid" - histogram = Histogram{} - histogram.AddDiagnostic(testName, testGUID) - } - - s.T().Run("should do nothing if adding an identical name-guid pair", func(t *testing.T) { - setUp() - expectedDiagnostics := map[string]string{testName: testGUID} - - histogram.AddDiagnostic(testName, testGUID) - - if !reflect.DeepEqual(expectedDiagnostics, histogram.Diagnostics) { - s.T().Errorf("invalid diagnostics map. Expected %v. Got %v", - expectedDiagnostics, histogram.Diagnostics) - } - }) - - s.T().Run("should add a new name-guid pair when given a unique name.", func(t *testing.T) { - setUp() - expectedDiagnostics := map[string]string{ - testName: testGUID, - "different-name": testGUID, - } - - histogram.AddDiagnostic("different-name", testGUID) - - if !reflect.DeepEqual(expectedDiagnostics, histogram.Diagnostics) { - s.T().Errorf("invalid diagnostics map. Expected %v. Got %v", - expectedDiagnostics, histogram.Diagnostics) - } - }) - - s.T().Run("should overwrite an existing name-guid pair", func(t *testing.T) { - setUp() - expectedDiagnostics := map[string]string{ - testName: "different-guid", - } - - histogram.AddDiagnostic(testName, "different-guid") - if !reflect.DeepEqual(expectedDiagnostics, histogram.Diagnostics) { - s.T().Errorf("Diagnostics map was modified. Expected %v. Got %v", - expectedDiagnostics, histogram.Diagnostics) - } - }) -}
diff --git a/cmd/catapult/main.go b/cmd/catapult/main.go deleted file mode 100644 index 54d04d0..0000000 --- a/cmd/catapult/main.go +++ /dev/null
@@ -1,21 +0,0 @@ -// Copyright 2017 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package main - -import ( - "context" - "flag" - "os" - - "github.com/google/subcommands" -) - -func main() { - subcommands.Register(&MakeHistogramCommand{}, "") - subcommands.Register(&UploadCommand{}, "") - - flag.Parse() - os.Exit(int(subcommands.Execute(context.Background()))) -}
diff --git a/cmd/catapult/make_histogram.go b/cmd/catapult/make_histogram.go deleted file mode 100644 index d52b21b..0000000 --- a/cmd/catapult/make_histogram.go +++ /dev/null
@@ -1,177 +0,0 @@ -// Copyright 2017 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - - "fuchsia.googlesource.com/testing/catapult" - schema "fuchsia.googlesource.com/testing/perf/schema/v1" - "github.com/google/subcommands" - uuid "github.com/satori/go.uuid" -) - -// sharedDiagnostics contains the values to use in the HistogramSet generated -// by this subcommand. -// -// Catapult requires a specific set of shared diagnostics to be present on a -// HistogramSet before uploading to the dashboard. Those required diagnostics -// are: -// -// * bots - a GenericSet of strings containing bot hostnames -// * benchmarks - a GenericSet of strings containing Telemetry -// benchmark names. -// * chromiumCommitPositions - a GenericSet of numbers containing -// Chromium commit positions. -// * masters - a GenericSet of strings containing buildbot master hostnames. -// -// There are other, optional diagnostics. Those may be added to this struct as -// needed. -// -// Because the names of some diagnostics are not relevant to users of this tool -// it is ok for this subcommand and this struct to use different names for any -// of them so long as the names are mapped back to their counterparts when the -// HistogramSet is constructed. -// -// To learn more about Catapult Diagnostics, see: -// https://github.com/catapult-project/catapult/blob/master/docs/how-to-write-metrics.md -type sharedDiagnostics struct { - // testSuite specifies the test suite that we are creating a - // HistogramSet for. e.g. "zircon" "ledger" - // - // Corresponds to the diagnostic "benchmarks". - testSuite string - - // builder specifies the LUCI builder that ran these tests. - // - // Corresponds to the diagnostic "bots". - builder string - - // bucket specifies the LUCI bucket containing the job that ran - // these tests. - // - // Corresponds to the diagnostic "masters". - bucket string - - // dateTime marks the time when tests were executed in ms since the UNIX - // Epoch. The Catapult dashboard uses this value to order sample points in - // a graph. - // - // Corresponds to the diagnostic "chromiumCommitPositions". - dateTime uint64 -} - -type MakeHistogramCommand struct { - sharedDiagnostics sharedDiagnostics -} - -func (*MakeHistogramCommand) Name() string { - return "make_histogram" -} - -func (*MakeHistogramCommand) Usage() string { - return "make_histogram [options] input_file" -} - -func (*MakeHistogramCommand) Synopsis() string { - return "Converts performance test output to a catapult HistogramSet" -} - -func (cmd *MakeHistogramCommand) SetFlags(flags *flag.FlagSet) { - flags.StringVar(&cmd.sharedDiagnostics.testSuite, "test-suite", "", - "Test suite corresponding to the input file") - flags.StringVar(&cmd.sharedDiagnostics.builder, "builder", "", - "LUCI builder that generated the test results") - flags.StringVar(&cmd.sharedDiagnostics.bucket, "bucket", "", - "Buildbucket bucket containing the job that ran the test") - flags.Uint64Var(&cmd.sharedDiagnostics.dateTime, - "datetime", 0, - "The date (in ms since epoch) the given tests were executed") -} - -// Execute converts performance test output to a catapult HistogramSet. -// -// A shared diagnostic will appear in the resulting HistogramSet for each -// diagnostic flag specified when invoking this subcommand. -// -// For more information about the HistogramSet format, See -// https://github.com/catapult-project/catapult/blob/master/docs/histogram-set-json-format.md -func (cmd *MakeHistogramCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - if f.NArg() != 1 { - log.Println("missing input_file") - return subcommands.ExitFailure - } - - inputFile := f.Arg(0) - blob, err := ioutil.ReadFile(inputFile) - if err != nil { - log.Println(err) - return subcommands.ExitFailure - } - - histogramSet := new(catapult.HistogramSet) - - var variants struct { - Variants []schema.Variant `json:"variants"` - } - - if err := json.Unmarshal(blob, &variants); err != nil { - log.Println(err) - return subcommands.ExitFailure - } - - // Generate Histograms from test results. - histograms := catapult.ConvertVariantsToHistograms(variants.Variants) - - // Track whether any single Histogram addition failed. We do this instead of - // exiting after a single error to expose as many errors as possible. - anyAddFailed := false - for _, histogram := range histograms { - if err := histogramSet.AddHistogram(histogram); err != nil { - log.Println(err) - anyAddFailed = true - } - } - if anyAddFailed { - return subcommands.ExitFailure - } - - // Add all shared diagnostics, mapping to the appropriate Catapult - // diagnostic names. - diagnosticInfoPairs := []struct { - Name string - Value interface{} - }{ - // chromiumCommitPositions must come first or Catapult will reject the upload. - {"chromiumCommitPositions", cmd.sharedDiagnostics.dateTime}, - {"benchmarks", cmd.sharedDiagnostics.testSuite}, - {"bots", cmd.sharedDiagnostics.builder}, - {"masters", cmd.sharedDiagnostics.bucket}, - } - for _, info := range diagnosticInfoPairs { - histogramSet.AddSharedDiagnostic(info.Name, &catapult.GenericSetDiagnostic{ - Type: catapult.DiagnosticTypeGenericSet, - GUID: uuid.NewV4().String(), - Values: []interface{}{info.Value}, - }) - } - - // Serialize and log the new HistogramSet. - js, err := json.Marshal(histogramSet.ToJSON()) - if err != nil { - log.Println(err) - return subcommands.ExitFailure - } - - // TODO(kjharland): Add option to write to a file so this can be used - // from a recipe. - fmt.Println(string(js)) - return subcommands.ExitSuccess -}
diff --git a/cmd/catapult/upload.go b/cmd/catapult/upload.go deleted file mode 100644 index d516532..0000000 --- a/cmd/catapult/upload.go +++ /dev/null
@@ -1,159 +0,0 @@ -// Copyright 2018 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file - -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/google/subcommands" - "go.chromium.org/luci/auth" - "go.chromium.org/luci/auth/client/authcli" -) - -// UploadCommand uploads a data file to a URL when executed. -type UploadCommand struct { - // The timeout for HTTP requests. - timeout time.Duration - - // The URL to upload data to - url string - - // LUCI flags used to parse command-line authentication options. - authFlags authcli.Flags -} - -func (*UploadCommand) Name() string { - return "upload" -} - -func (*UploadCommand) Usage() string { - return "upload [options] json_file" -} - -func (*UploadCommand) Synopsis() string { - return "Uploads a JSON file to a URL" -} - -func (cmd *UploadCommand) SetFlags(flags *flag.FlagSet) { - cmd.authFlags = authcli.Flags{} - cmd.authFlags.Register(flags, auth.Options{}) - flags.DurationVar(&cmd.timeout, "timeout", 10*time.Second, - "Request timeout duration string. e.g. 12s or 1m") - flags.StringVar(&cmd.url, "url", "", "(required) The URL to upload data to") -} - -func (cmd *UploadCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - if f.NArg() < 1 { - fmt.Fprintln(os.Stderr, "missing input file") - return subcommands.ExitFailure - } - if f.NArg() != 1 { - fmt.Fprintln(os.Stderr, "too many positional arguments") - return subcommands.ExitFailure - } - if len(cmd.url) == 0 { - fmt.Fprintln(os.Stderr, "url is required") - return subcommands.ExitFailure - } - if cmd.timeout <= 0 { - fmt.Fprintf(os.Stderr, "timeout must be positive. Got %v\n", cmd.timeout) - return subcommands.ExitFailure - } - - opts, err := cmd.authFlags.Options() - if err != nil { - fmt.Fprintln(os.Stderr, err) - return subcommands.ExitFailure - } - - inputFile := f.Arg(0) - if err := uploadData(ctx, cmd.url, inputFile, cmd.timeout, opts); err != nil { - fmt.Fprintln(os.Stderr, err) - return subcommands.ExitFailure - } - - return subcommands.ExitSuccess -} - -func uploadData(ctx context.Context, catapultURL string, filepath string, timeout time.Duration, authOpts auth.Options) error { - requestBody, err := ioutil.ReadFile(filepath) - if err != nil { - return err - } - - // Verify that the input data is actual JSON. Ideally we'd verify the - // structure of the data also but this isn't always practical. (For - // example, when parsing a dynamic schema such as Catapult's - // HistogramSet). - // TODO(kjharland): use json.Valid after fuchsia's go is updated to v1.10 - var ignored interface{} - if err := json.Unmarshal(requestBody, &ignored); err != nil { - return fmt.Errorf("input is not valid JSON: %v", err) - } - - // URL encode the request body. This is weird but the Catapult dashboard - // will reject the request because it expects a body of the form: - // `data=url-encoded-histogramset` instead of the more usual JSON body: - // `{data: ...}`. - params := url.Values{} - params.Set("data", string(requestBody)) - content := params.Encode() - - req, err := http.NewRequest(http.MethodPost, catapultURL, strings.NewReader(content)) - if err != nil { - return fmt.Errorf("create request: %v", err.Error()) - } - // The User-Agent header helps distinguish this uploader's entries from - // other entries in the Catapult logs. If left empty, Go inserts its - // own header "Go-http-client/XX" where XX is the version of the agent. - // This is indistinguishable from other Go http clients. - req.Header.Set("User-Agent", "Fuchsia-Uploader/1.0") - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Content-Length", strconv.Itoa(len(content))) - - // The LUCI authenticator used to authenticate the request. Silent login - // prevents an interactive login session when running from CI bots. - authenticator := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts) - - // Generate the OAuth token used to authenticate the request. - oauthToken, err := authenticator.GetAccessToken(time.Minute) - if err != nil { - return fmt.Errorf("create token: %v", err.Error()) - } - - // Set an auth header on the request, containing the token generated from the - // provided service account information. - oauthToken.SetAuthHeader(req) - - client, err := authenticator.Client() - if err != nil { - return err - } - client.Timeout = timeout - - response, err := client.Do(req) - if err != nil { - return err - } - - if response.StatusCode < 200 || response.StatusCode >= 300 { - return errors.New(response.Status) - } - - log.Println(response.Status) - return nil -}
diff --git a/manifest b/manifest deleted file mode 100644 index d731908..0000000 --- a/manifest +++ /dev/null
@@ -1,8 +0,0 @@ -<manifest> - <projects> - <project name="testing" - path="go/src/fuchsia.googlesource.com/testing" - remote="https://fuchsia.googlesource.com/testing" - gerrithost="https://fuchsia-review.googlesource.com"/> - </projects> -</manifest>
diff --git a/perf/schema/v1/schema.go b/perf/schema/v1/schema.go deleted file mode 100644 index 4f625a2..0000000 --- a/perf/schema/v1/schema.go +++ /dev/null
@@ -1,27 +0,0 @@ -// Copyright 2017 The Fuchsia Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package schema defines the schema for Fuchsia's performance test output. -package schema - -// Variant is a collection of benchmark runs with a specified configuration. -type Variant struct { - FinishTime int64 `json:"finish_time"` - FBenchmarksData []BenchmarkData `json:"fbenchmarks_data"` - VariantDesc string `json:"variant_desc"` - BootArgs []string `json:"boot_args"` -} - -// BenchmarkData is a single benchmark. -type BenchmarkData struct { - Label string `json:"label"` - Unit string `json:"unit"` - Samples []Sample `json:"samples"` -} - -// Sample is a samping of multiple values from a single benchmark. -type Sample struct { - Label string `json:"label"` - Values []float64 `json:"values"` -}