[testsharder] Begin soft transition s/label/tags
Why:
(1) "label" clashes with the GN label
(2) tags = plural; we will need arbitrary tags/labels to support things
like product configurations
Change-Id: I90fc87b3e68fc99bb8f40d2185458198806cd34b
diff --git a/cmd/testsharder/main.go b/cmd/testsharder/main.go
index 62fb2fa..245e6c6 100644
--- a/cmd/testsharder/main.go
+++ b/cmd/testsharder/main.go
@@ -9,6 +9,7 @@
"log"
"os"
+ "fuchsia.googlesource.com/tools/command"
"fuchsia.googlesource.com/tools/testsharder"
)
@@ -19,14 +20,14 @@
// The filepath to write output to. If unspecified, stdout is used.
outputFile string
- // Label is a key on which to filter environments, which are labeled.
- label string
+ // Tags are keys on which to filter environments, which are labeled.
+ tags command.StringsFlag
)
func init() {
flag.StringVar(&buildDir, "build-dir", "", "path to the fuchsia build directory root (required)")
flag.StringVar(&outputFile, "output-file", "", "path to a file which will contain the shards as JSON, default is stdout")
- flag.StringVar(&label, "label", "", "environment label on which to filter")
+ flag.Var(&tags, "tag", "environment tags on which to filter; only the tests that match all tags will be sharded")
}
func main() {
@@ -51,7 +52,7 @@
}
// Create shards and write them to an output file if specifed, else stdout.
- shards := testsharder.MakeShards(specs, label)
+ shards := testsharder.MakeShards(specs, tags)
f := os.Stdout
if outputFile != "" {
var err error
diff --git a/testsharder/environment.go b/testsharder/environment.go
index d0b2226..c9a4e76 100644
--- a/testsharder/environment.go
+++ b/testsharder/environment.go
@@ -24,7 +24,12 @@
Dimensions DimensionSet `json:"dimensions"`
// Label is a label given to an environment on which the testsharder may filter.
+ //
+ // TODO(joshuaseaton): deprecate in favor of tags.
Label string `json:"label,omitempty"`
+
+ // Tags are keys given to an environment on which the testsharder may filter.
+ Tags []string `json:"tags,omitempty"`
}
// Name returns a name calculated from its specfied properties.
diff --git a/testsharder/shard.go b/testsharder/shard.go
index ea70c9b..a743cd5 100644
--- a/testsharder/shard.go
+++ b/testsharder/shard.go
@@ -4,6 +4,7 @@
package testsharder
import (
+ "fmt"
"sort"
)
@@ -20,30 +21,42 @@
}
// MakeShards is the core algorithm to this tool. It takes a set of test specs and produces
-// a set of shards which may then be converted into Swarming tasks. Environments with a
-// label different from the provided will be ignored.
+// a set of shards which may then be converted into Swarming tasks.
+//
+// Environments that do not match all specified tags will be ignored.
//
// This is the most naive algorithm at the moment. It just merges all tests together which
// have the same environment setting into the same shard.
-func MakeShards(specs []TestSpec, label string) []*Shard {
+func MakeShards(specs []TestSpec, tags []string) []*Shard {
// Collect the order of the shards so our shard ordering is deterministic with
// respect to the input.
- envToSuites := make(map[Environment][]TestSpec)
+ envToSuites := newEnvMap()
envs := []Environment{}
for _, spec := range specs {
for _, env := range spec.Envs {
- if env.Label != label {
+ envTags := env.Tags
+ if env.Label != "" {
+ envTags = append(envTags, env.Label)
+ }
+ if !stringSlicesEq(tags, envTags) {
continue
}
- if _, ok := envToSuites[env]; !ok {
+
+ // Tags should not differ by ordering.
+ sortableTags := sort.StringSlice(tags)
+ sortableTags.Sort()
+ env.Tags = []string(sortableTags)
+
+ specs, ok := envToSuites.get(env)
+ if !ok {
envs = append(envs, env)
}
- envToSuites[env] = append(envToSuites[env], spec)
+ envToSuites.set(env, append(specs, spec))
}
}
shards := make([]*Shard, 0, len(envs))
for _, env := range envs {
- specs := envToSuites[env]
+ specs, _ := envToSuites.get(env)
sort.Slice(specs, func(i, j int) bool {
return specs[i].Test.Path < specs[i].Test.Path
})
@@ -59,3 +72,39 @@
}
return shards
}
+
+// Abstracts a mapping Environment -> []string, as Environment contains non-comparable
+// members (e.g., string slices), which makes it invalid for a map key.
+type envMap struct {
+ m map[string][]TestSpec
+}
+
+func newEnvMap() envMap {
+ return envMap{m: make(map[string][]TestSpec)}
+}
+
+func (em envMap) get(e Environment) ([]TestSpec, bool) {
+ specs, ok := em.m[fmt.Sprintf("%v", e)]
+ return specs, ok
+}
+
+func (em *envMap) set(e Environment, specs []TestSpec) {
+ em.m[fmt.Sprintf("%v", e)] = specs
+}
+
+func stringSlicesEq(s []string, t []string) bool {
+ if len(s) != len(t) {
+ return false
+ }
+ seen := make(map[string]int)
+ for i, _ := range s {
+ seen[s[i]] += 1
+ seen[t[i]] -= 1
+ }
+ for _, v := range seen {
+ if v != 0 {
+ return false
+ }
+ }
+ return true
+}
diff --git a/testsharder/shard_test.go b/testsharder/shard_test.go
index 4bc3ec5..e9ee45c 100644
--- a/testsharder/shard_test.go
+++ b/testsharder/shard_test.go
@@ -1,106 +1,123 @@
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-package testsharder_test
+package testsharder
import (
"fmt"
"reflect"
"testing"
-
- "fuchsia.googlesource.com/tools/testsharder"
)
// Note that just printing a list of shard pointers will print a list of memory addresses,
// which would make for an unhelpful error message.
-func assertEqual(t *testing.T, expected, actual []*testsharder.Shard) {
+func assertEqual(t *testing.T, expected, actual []*Shard) {
if !reflect.DeepEqual(expected, actual) {
- errMsg := "expected:\n"
+ errMsg := "\nexpected:\n"
for _, shard := range expected {
- errMsg += fmt.Sprintf("%+v,\n", shard)
+ errMsg += fmt.Sprintf("%v,\n", shard)
}
- errMsg += "actual:\n"
+ errMsg += "\nactual:\n"
for _, shard := range actual {
- errMsg += fmt.Sprintf("%+v,\n", shard)
+ errMsg += fmt.Sprintf("%v,\n", shard)
}
t.Fatalf(errMsg)
}
}
func TestMakeShards(t *testing.T) {
- test1 := testsharder.Test{
- Location: "/path/to/binary",
+ env1 := Environment{
+ Dimensions: DimensionSet{DeviceType: "QEMU"},
+ Tags: []string{},
}
- test2 := testsharder.Test{
- Location: "/path/to/binary2",
+ env2 := Environment{
+ Dimensions: DimensionSet{DeviceType: "NUC"},
+ Tags: []string{},
}
- env1 := testsharder.Environment{
- Dimensions: testsharder.DimensionSet{DeviceType: "QEMU"},
+ env3 := Environment{
+ Dimensions: DimensionSet{OS: "Linux"},
+ Tags: []string{},
}
- env2 := testsharder.Environment{
- Dimensions: testsharder.DimensionSet{DeviceType: "NUC"},
- }
- env3 := testsharder.Environment{
- Dimensions: testsharder.DimensionSet{DeviceType: "NUC"},
- Label: "LABEL",
- }
- spec1 := testsharder.TestSpec{
- Test: test1,
- Envs: []testsharder.Environment{env1},
- }
- spec2 := testsharder.TestSpec{
- Test: test1,
- Envs: []testsharder.Environment{env1},
- }
- spec3 := testsharder.TestSpec{
- Test: test2,
- Envs: []testsharder.Environment{env1, env2},
- }
- spec4 := testsharder.TestSpec{
- Test: test2,
- Envs: []testsharder.Environment{env1, env2, env3},
- }
- t.Run("Ensure that environments have names", func(t *testing.T) {
- // This will in turn ensure that the associated shards too have
- // names.
- if env1.Name() == "" {
- t.Fatalf("Environment\n%+v\n has an empty name", env1)
- }
- if env2.Name() == "" {
- t.Fatalf("Environment\n%+v\n has an empty name", env2)
+ t.Run("environments have nonempty names", func(t *testing.T) {
+ envs := []Environment{env1, env2, env3}
+ for _, env := range envs {
+ if env.Name() == "" {
+ t.Fatalf("Environment\n%+v\n has an empty name", env)
+ }
}
})
- t.Run("Ensure env shared", func(t *testing.T) {
- actual := testsharder.MakeShards([]testsharder.TestSpec{spec1, spec2, spec3}, "")
- expected := []*testsharder.Shard{
- // Ensure that the order of the shards is the order in which their
- // corresponding environments appear in the input. This is the simplest
- // deterministic order we can produce for the shards.
- {
- Name: env1.Name(),
- // Ensure that we actually specify the test _twice_, that is, don't
- // necessarily deduplicate tests for a shared environment.
- Tests: []testsharder.Test{test1, test1, test2},
- Env: env1,
+ spec := func(id int, envs ...Environment) TestSpec {
+ return TestSpec{
+ Test: Test{
+ Location: fmt.Sprintf("/path/to/test/%d", id),
},
- {
- Name: env2.Name(),
- Tests: []testsharder.Test{test2},
- Env: env2,
- },
+ Envs: envs,
}
+ }
+
+ shard := func(env Environment, ids ...int) *Shard {
+ var tests []Test
+ for _, id := range ids {
+ tests = append(tests, spec(id, env).Test)
+ }
+ return &Shard{
+ Name: env.Name(),
+ Tests: tests,
+ Env: env,
+ }
+ }
+
+ t.Run("tests of same environment are grouped", func(t *testing.T) {
+ actual := MakeShards(
+ []TestSpec{spec(1, env1, env2), spec(2, env1, env3), spec(3, env3)},
+ []string{},
+ )
+ expected := []*Shard{shard(env1, 1, 2), shard(env2, 1), shard(env3, 2, 3)}
assertEqual(t, expected, actual)
})
- t.Run("Ensure label respected", func(t *testing.T) {
- actual := testsharder.MakeShards([]testsharder.TestSpec{spec1, spec2, spec3, spec4}, env3.Label)
- expected := []*testsharder.Shard{
- {
- Name: env3.Name(),
- Tests: []testsharder.Test{test2},
- Env: env3,
+ t.Run("there is no deduplication of tests", func(t *testing.T) {
+ actual := MakeShards(
+ []TestSpec{spec(1, env1), spec(1, env1), spec(1, env1)},
+ []string{},
+ )
+ expected := []*Shard{shard(env1, 1, 1, 1)}
+ assertEqual(t, expected, actual)
+ })
+
+ // Ensure that the order of the shards is the order in which their
+ // corresponding environments appear in the input. This is the simplest
+ // deterministic order we can produce for the shards.
+ t.Run("shards are ordered", func(t *testing.T) {
+ actual := MakeShards(
+ []TestSpec{spec(1, env2, env3), spec(2, env1), spec(3, env3)},
+ []string{},
+ )
+ expected := []*Shard{shard(env2, 1), shard(env3, 1, 3), shard(env1, 2)}
+ assertEqual(t, expected, actual)
+ })
+
+ t.Run("tags are respected", func(t *testing.T) {
+ tagger := func(env Environment, tags ...string) Environment {
+ env2 := env
+ env2.Tags = tags
+ return env2
+ }
+
+ actual := MakeShards(
+ []TestSpec{
+ spec(1, tagger(env1, "A")),
+ spec(2, tagger(env1, "A", "B", "C")),
+ spec(3, tagger(env2, "B", "C")),
+ spec(4, tagger(env3, "C", "A")),
+ spec(5, tagger(env3, "A", "C")),
},
+ []string{"A", "C"},
+ )
+ expected := []*Shard{
+ // "C", "A" and "A", "C" should define the same tags.
+ shard(tagger(env3, "A", "C"), 4, 5),
}
assertEqual(t, expected, actual)
})
