fuchsia/testexec/test_spec.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package testexec

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"fuchsia.googlesource.com/infra/infra/fuchsia"
)

// These are the default environments to be used if an author specifies none.
//
// TODO(IN-497) We include the NUC here as a tranistional policy. We will give
// people a week from flag day to begin explicitly specifying their
// environments, after which we will set the default as only QEMU.
var defaultEnvs = []Environment{
	Environment{
		Device: DeviceSpec{
			Type: "QEMU",
		},
	},
	Environment{
		Device: DeviceSpec{
			Type: "Intel NUC Kit NUC7i5DNHE",
		},
	},
}

// TestSpec is the specification for a single test and the environments it
// should be executed in.
type TestSpec struct {
	// Test is the test that this specuration is for.
	Test `json:"test"`

	// Envs is a set of environments that the test should be executed in.
	Envs []Environment `json:"environments"`
}

// Test encapsulates details about a particular test.
type Test struct {
	// Name is the full, GN source-relative target name of the test
	// (e.g., //garnet/bin/foo/tests:foo_tests).
	Name string `json:"name"`

	// Location is a unique reference to a test: for example, a filesystem
	// path or a Fuchsia URI.
	Location string `json:"location"`
}

func (spec TestSpec) validateAgainst(platforms []Environment) error {
	if spec.Test.Name == "" {
		return fmt.Errorf("A test spec's test must have a non-empty name")
	}
	if spec.Test.Location == "" {
		return fmt.Errorf("A test spec's test must have a non-empty location")
	}

	resolvesToOneOf := func(env Environment, platforms []Environment) bool {
		for _, platform := range platforms {
			if env.resolvesTo(platform) {
				return true
			}
		}
		return false
	}

	var badEnvs []Environment
	for _, env := range spec.Envs {
		if !resolvesToOneOf(env, platforms) {
			badEnvs = append(badEnvs, env)
		}
	}
	if len(badEnvs) > 0 {
		return fmt.Errorf(
			`the following environments of test\n%+v were malformed
			or did not match any available test platforms:\n%+v`,
			spec.Test, badEnvs)
	}
	return nil
}

// ValidateTestSpecs validates a list of test specs against a list of test
// platform environments.
func ValidateTestSpecs(specs []TestSpec, platforms []Environment) error {
	errMsg := ""
	for _, spec := range specs {
		if err := spec.validateAgainst(platforms); err != nil {
			errMsg += fmt.Sprintf("\n%v", err)
		}
	}
	if errMsg != "" {
		return fmt.Errorf(errMsg)
	}
	return nil
}

// HasTestSpecExt returns true if the given path ends with .spec.json.
func HasTestSpecExt(path string) bool {
	if first := filepath.Ext(path); first != ".json" {
		return false
	}
	if second := filepath.Ext(path[:len(path)-len(".json")]); second != ".spec" {
		return false
	}
	return true
}

// LoadTestSpecs loads a set of test specifications from a manifest of Fuchsia packages.
func LoadTestSpecs(fuchsiaBuildDir string, pkgManifest fuchsia.PackageManifest) ([]TestSpec, error) {
	// Use the packages manifest to discover test specifications.
	//
	// It is guaranteed that a test spec will be written to the build directory of the
	// corresponding package its test was defined in: specifically, it will be put in
	// <target_out_dir of the test package>/<test package name>.
	// This algorithm iterates over these directories for each package in the package
	// manifest.
	specs := make([]TestSpec, 0, len(pkgManifest.Packages))
	for _, pkg := range pkgManifest.Packages {
		testSpecDir := filepath.Join(fuchsiaBuildDir, pkg.BuildDir, pkg.Name)

		// If the associated test spec directory does not exist, the package specified no
		// tests.
		if _, err := os.Stat(testSpecDir); os.IsNotExist(err) {
			continue
		}

		// Non-recursively enumerate the files in this directory; it's guaranteed that
		// the test specs will be found here if generated.
		entries, err := ioutil.ReadDir(testSpecDir)
		if err != nil {
			return nil, err
		}

		for _, entry := range entries {
			path := filepath.Join(testSpecDir, entry.Name())

			// Skip any directories or files without a .spec.json extension.
			// At this time, only test specs are generated in this subdirectory.
			if entry.IsDir() || !HasTestSpecExt(path) {
				continue
			}

			// Open, read, and parse the test spec. If at any point we fail from here,
			// fail the whole algorithm, since either the file is malformed which
			// suggests larger problems (e.g. a file from the build is impersonating
			// a test spec accidentally).
			f, err := os.Open(path)
			if err != nil {
				return nil, fmt.Errorf("failed to open %s: %s", path, err.Error())
			}
			defer f.Close()

			var spec TestSpec
			if err := json.NewDecoder(f).Decode(&spec); err != nil {
				return nil, fmt.Errorf("failed to decode %s: %s", path, err.Error())
			}
			if len(spec.Envs) == 0 {
				spec.Envs = defaultEnvs
			}
			specs = append(specs, spec)
		}
	}
	return specs, nil
}