fuchsia/testexec/test_spec_test.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package testexec_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"testing"

	"fuchsia.googlesource.com/infra/infra/fuchsia"
	"fuchsia.googlesource.com/infra/infra/fuchsia/testexec"
)

var qemuEnv = testexec.Environment{
	Device: testexec.DeviceSpec{
		Type: "QEMU",
	},
}

var nucEnv = testexec.Environment{
	Device: testexec.DeviceSpec{
		Type: "NUC",
	},
}

var specFoo1 = testexec.TestSpec{
	Test: testexec.Test{
		Name:     "//obsidian/bin/foo:foo_unittests",
		Location: "/system/test/foo_unittests",
	},
	Envs: []testexec.Environment{qemuEnv},
}

var specFoo2 = testexec.TestSpec{
	Test: testexec.Test{
		Name:     "//obsidian/bin/foo:foo_integration_tests",
		Location: "/system/test/foo_integration_tests",
	},
	Envs: []testexec.Environment{qemuEnv, nucEnv},
}

var specBar = testexec.TestSpec{
	Test: testexec.Test{
		Name:     "//obsidian/lib/bar:bar_tests",
		Location: "/system/test/bar_tests",
	},
	Envs: []testexec.Environment{qemuEnv},
}

// FuchsiaBuildDir is a struct representing the root build directory of a fuchsia
// checkout.
type fuchsiaBuildDir struct {
	root string
	t    *testing.T
}

func newFuchsiaBuildDir(t *testing.T) *fuchsiaBuildDir {
	root, err := ioutil.TempDir("", "fuchsia-build-dir")
	if err != nil {
		t.Fatalf("could not create fuchsia build directory: %v", err)
	}
	return &fuchsiaBuildDir{
		root: root,
		t:    t,
	}
}

// CreatePackageLayout takes a package manifest and creates the associated
// directory layout.
func (bd fuchsiaBuildDir) createPackageLayout(pkgManifest fuchsia.PackageManifest) {
	for _, pkg := range pkgManifest.Packages {
		specDir := filepath.Join(bd.root, pkg.BuildDir, pkg.Name)
		if err := os.MkdirAll(specDir, os.ModePerm); err != nil {
			bd.t.Fatalf("could not create test spec directory for package \"%s\": %v", pkg.Name, err)
		}
	}
}

func writeTestSpec(t *testing.T, spec testexec.TestSpec, path string) {
	bytes, err := json.Marshal(&spec)
	if err != nil {
		t.Fatal(err)
	}
	if err = ioutil.WriteFile(path, bytes, os.ModePerm); err != nil {
		t.Fatalf("could not write test spec to %s: %v", path, err)
	}
}

func TestLoadTestSpecs(t *testing.T) {
	areEqual := func(a, b []testexec.TestSpec) bool {
		stringify := func(spec testexec.TestSpec) string {
			return fmt.Sprintf("%#v", spec)
		}
		sort := func(list []testexec.TestSpec) {
			sort.Slice(list[:], func(i, j int) bool {
				return stringify(list[i]) < stringify(list[j])
			})
		}
		sort(a)
		sort(b)
		return reflect.DeepEqual(a, b)
	}

	packageFoo := "foo"
	buildDirFoo := "obj/obsidian/bin/foo"
	packageBar := "bar"
	buildDirBar := "obj/obsidian/lib/bar"

	pkgManifest := fuchsia.PackageManifest{
		Packages: []fuchsia.PackageDescriptor{
			{
				BuildDir: buildDirFoo,
				Name:     packageFoo,
			},
			{
				BuildDir: buildDirBar,
				Name:     packageBar,
			},
		},
	}

	correctSpecsLoad := func(t *testing.T, expected []testexec.TestSpec, fuchsiaBuildDir string) {
		actual, err := testexec.LoadTestSpecs(fuchsiaBuildDir, pkgManifest)
		if err != nil {
			t.Fatalf("error while loading test specs: %v", err)
		}
		if !areEqual(expected, actual) {
			t.Fatalf("test specs not properly loaded:\nexpected:\n%+v\nactual:\n%+v", expected, actual)
		}
	}

	t.Run("test specs are found", func(t *testing.T) {
		bd := newFuchsiaBuildDir(t)
		defer os.RemoveAll(bd.root)
		bd.createPackageLayout(pkgManifest)

		specDirFoo := filepath.Join(bd.root, buildDirFoo, packageFoo)
		specDirBar := filepath.Join(bd.root, buildDirBar, packageBar)

		writeTestSpec(t, specFoo1, filepath.Join(specDirFoo, "foo_unittests.spec.json"))
		writeTestSpec(t, specFoo2, filepath.Join(specDirFoo, "foo_integration_tests.spec.json"))
		writeTestSpec(t, specBar, filepath.Join(specDirBar, "bar_tests.spec.json"))

		expected := []testexec.TestSpec{specFoo1, specFoo2, specBar}
		correctSpecsLoad(t, expected, bd.root)
	})

	t.Run("test specs in wrong location are ignored", func(t *testing.T) {
		bd := newFuchsiaBuildDir(t)
		defer os.RemoveAll(bd.root)
		bd.createPackageLayout(pkgManifest)

		specDirFoo := filepath.Join(bd.root, buildDirFoo, packageFoo)
		specDirBar := filepath.Join(bd.root, buildDirBar, packageBar)
		nonSpecDir := filepath.Join(bd.root, buildDirBar, "other-package")
		if err := os.MkdirAll(nonSpecDir, os.ModePerm); err != nil {
			t.Fatalf("failed to create a directory outside of the package manifest: %v", err)
		}

		writeTestSpec(t, specFoo1, filepath.Join(specDirFoo, "foo_unittests.spec.json"))
		writeTestSpec(t, specFoo2, filepath.Join(nonSpecDir, "other_tests.spec.json"))
		writeTestSpec(t, specBar, filepath.Join(specDirBar, "bar_tests.spec.json"))

		expected := []testexec.TestSpec{specFoo1, specBar}
		correctSpecsLoad(t, expected, bd.root)
	})

	t.Run("test specs with wrong extension are ignored", func(t *testing.T) {
		bd := newFuchsiaBuildDir(t)
		defer os.RemoveAll(bd.root)
		bd.createPackageLayout(pkgManifest)

		specDirFoo := filepath.Join(bd.root, buildDirFoo, packageFoo)
		specDirBar := filepath.Join(bd.root, buildDirBar, packageBar)

		writeTestSpec(t, specFoo1, filepath.Join(specDirFoo, "bad_extension1.json"))
		writeTestSpec(t, specFoo2, filepath.Join(specDirFoo, "good extension.spec.json"))
		writeTestSpec(t, specBar, filepath.Join(specDirBar, "bad_extension2.spec"))

		expected := []testexec.TestSpec{specFoo2}
		correctSpecsLoad(t, expected, bd.root)
	})

	t.Run("malformed test specs raise error", func(t *testing.T) {
		bd := newFuchsiaBuildDir(t)
		defer os.RemoveAll(bd.root)
		bd.createPackageLayout(pkgManifest)

		specDirFoo := filepath.Join(bd.root, buildDirFoo, packageFoo)
		specDirBar := filepath.Join(bd.root, buildDirBar, packageBar)

		writeTestSpec(t, specFoo1, filepath.Join(specDirFoo, "foo_unittests.spec.json"))
		if err := ioutil.WriteFile(filepath.Join(specDirFoo, "foo_integration_tests.spec.json"),
			[]byte("{I am not a test spec}"), os.ModePerm); err != nil {
			t.Fatalf("could not write malformed test spec: %v", err)
		}
		writeTestSpec(t, specBar, filepath.Join(specDirBar, "bar_tests.spec.json"))

		_, err := testexec.LoadTestSpecs(bd.root, pkgManifest)
		if err == nil {
			t.Fatalf("malformed test spec did not raise an error")
		}
	})
}

func TestValidateTestSpecs(t *testing.T) {
	noTestNameSpec := testexec.TestSpec{
		Test: testexec.Test{
			Location: "/system/test/baz_tests",
		},
		Envs: []testexec.Environment{qemuEnv},
	}
	noTestLocationSpec := testexec.TestSpec{
		Test: testexec.Test{
			Name: "//obsidian/public/lib/baz:baz_tests",
		},
		Envs: []testexec.Environment{qemuEnv},
	}
	badEnvSpec := testexec.TestSpec{
		Test: testexec.Test{
			Name:     "//obsidian/public/lib/baz:baz_tests",
			Location: "/system/test/baz_tests",
		},
		Envs: []testexec.Environment{
			testexec.Environment{
				Device: testexec.DeviceSpec{
					Type: "NON-EXISTENT-DEVICE",
				},
			},
		},
	}
	platforms := []testexec.Environment{qemuEnv, nucEnv}

	t.Run("valid specs are validated", func(t *testing.T) {
		validSpecLists := [][]testexec.TestSpec{
			{specFoo1}, {specFoo2}, {specBar},
			{specFoo1, specFoo2}, {specFoo1, specBar}, {specFoo2, specBar},
			{specFoo1, specFoo2, specBar},
		}
		for _, list := range validSpecLists {
			if err := testexec.ValidateTestSpecs(list, platforms); err != nil {
				t.Fatalf("valid specs marked as invalid: %+v: %v", list, err)
			}
		}
	})

	t.Run("invalid specs are invalidated", func(t *testing.T) {
		invalidSpecLists := [][]testexec.TestSpec{
			{noTestNameSpec}, {noTestLocationSpec}, {badEnvSpec},
			{noTestNameSpec, noTestLocationSpec}, {noTestNameSpec, badEnvSpec},
			{noTestLocationSpec, badEnvSpec},
			{noTestNameSpec, noTestLocationSpec, badEnvSpec},
		}
		for _, list := range invalidSpecLists {
			if err := testexec.ValidateTestSpecs(list, platforms); err == nil {
				t.Fatalf("invalid specs marked as valid: %+v", list)
			}
		}
	})
}