| // Copyright 2020 The Fuchsia Authors. All rights reserved. | 
 | // Use of this source code is governed by a BSD-style license that can be | 
 | // found in the LICENSE file. | 
 |  | 
 | package fuzz | 
 |  | 
 | import ( | 
 | 	"bufio" | 
 | 	"encoding/json" | 
 | 	"fmt" | 
 | 	"io" | 
 | 	"io/ioutil" | 
 | 	"os" | 
 | 	"path/filepath" | 
 | 	"regexp" | 
 | 	"runtime" | 
 | 	"strings" | 
 |  | 
 | 	"github.com/golang/glog" | 
 | ) | 
 |  | 
 | // A Build represents a Fuchsia build, consisting of all the resources needed | 
 | // to run a fuzzer on an instance (e.g. a Fuchsia image, fuzzer packages and | 
 | // metadata, binary symbols, support utilities, etc.). | 
 | type Build interface { | 
 | 	// Ensures all the needed resources are present and fetches any that are | 
 | 	// missing. Multiple calls to Prepare should be idempotent for the same | 
 | 	// Build. | 
 | 	Prepare() error | 
 |  | 
 | 	// Returns a fuzzer specified by a `package/binary` name, or an error if it | 
 | 	// isn't found. | 
 | 	Fuzzer(name string) (*Fuzzer, error) | 
 |  | 
 | 	// Returns the absolute host paths for each key.  Each key corresponds to a | 
 | 	// specific resource provided by the Build.  This abstraction allows for | 
 | 	// different build types to have different structures. | 
 | 	Path(keys ...string) ([]string, error) | 
 |  | 
 | 	// Reads input from `in`, symbolizes it, and writes it back to `out`. | 
 | 	// Returns on error, or when `in` has no more data to read.  Processing | 
 | 	// will be streamed, line-by-line. | 
 | 	// TODO(fxbug.dev/47482): does this belong elsewhere? | 
 | 	Symbolize(in io.ReadCloser, out io.Writer) error | 
 |  | 
 | 	// Returns a list of the names of the fuzzers that are available to run | 
 | 	ListFuzzers() []string | 
 | } | 
 |  | 
 | // BaseBuild is a simple implementation of the Build interface | 
 | type BaseBuild struct { | 
 | 	Fuzzers map[string]*Fuzzer | 
 | 	Paths   map[string]string | 
 | 	IDs     []string | 
 | } | 
 |  | 
 | // This is stubbed out to allow for test code to replace it | 
 | var NewBuild = NewBuildFromEnvironment | 
 |  | 
 | // This environment variable is set by the ClusterFuzz build manager | 
 | const clusterFuzzBundleDirEnvVar = "FUCHSIA_RESOURCES_DIR" | 
 |  | 
 | // Attempt to auto-detect the correct Build type | 
 | func NewBuildFromEnvironment() (Build, error) { | 
 | 	if _, found := os.LookupEnv(clusterFuzzBundleDirEnvVar); found { | 
 | 		return NewClusterFuzzLegacyBuild() | 
 | 	} | 
 | 	return NewLocalFuchsiaBuild() | 
 | } | 
 |  | 
 | // NewClusterFuzzLegacyBuild will create a BaseBuild with path layouts | 
 | // corresponding to the legacy build bundles used by ClusterFuzz's original | 
 | // Python integration. Note that these build bundles only support x64. | 
 | func NewClusterFuzzLegacyBuild() (Build, error) { | 
 | 	bundleDir, found := os.LookupEnv(clusterFuzzBundleDirEnvVar) | 
 | 	if !found { | 
 | 		return nil, fmt.Errorf("%s not set", clusterFuzzBundleDirEnvVar) | 
 | 	} | 
 |  | 
 | 	buildDir := filepath.Join(bundleDir, "build") | 
 | 	targetDir := filepath.Join(bundleDir, "target", "x64") | 
 | 	clangDir := filepath.Join(buildDir, "buildtools", "linux-x64", "clang") | 
 | 	build := &BaseBuild{ | 
 | 		Paths: map[string]string{ | 
 | 			"zbi":             filepath.Join(targetDir, "fuchsia.zbi"), | 
 | 			"fvm":             filepath.Join(buildDir, "out", "default.zircon", "tools", "fvm"), | 
 | 			"zbitool":         filepath.Join(buildDir, "out", "default.zircon", "tools", "zbi"), | 
 | 			"blk":             filepath.Join(targetDir, "fvm.blk"), | 
 | 			"qemu":            filepath.Join(bundleDir, "qemu-for-fuchsia", "bin", "qemu-system-x86_64"), | 
 | 			"kernel":          filepath.Join(targetDir, "multiboot.bin"), | 
 | 			"llvm-symbolizer": filepath.Join(clangDir, "bin", "llvm-symbolizer"), | 
 | 			"symbolizer":      filepath.Join(buildDir, "zircon", "prebuilt", "downloads", "symbolize", "linux-x64", "symbolize"), | 
 | 			"fuzzers.json":    filepath.Join(buildDir, "out", "default", "fuzzers.json"), | 
 | 		}, | 
 | 		IDs: []string{ | 
 | 			filepath.Join(clangDir, "lib", "debug", ".build_id"), | 
 | 			filepath.Join(buildDir, "out", "default", ".build-id"), | 
 | 			filepath.Join(buildDir, "out", "default.zircon", ".build-id"), | 
 | 		}, | 
 | 	} | 
 | 	if err := build.LoadFuzzers(); err != nil { | 
 | 		return nil, err | 
 | 	} | 
 |  | 
 | 	return build, nil | 
 | } | 
 |  | 
 | var Platforms = map[string]string{ | 
 | 	"linux":  "linux", | 
 | 	"darwin": "mac", | 
 | } | 
 |  | 
 | var Archs = map[string]struct { | 
 | 	Binary string | 
 | 	Kernel string | 
 | }{ | 
 | 	"x64":   {"qemu-system-x86_64", "multiboot.bin"}, | 
 | 	"arm64": {"qemu-system-aarch64", "qemu-boot-shim.bin"}, | 
 | } | 
 |  | 
 | var hostDir = map[string]string{"arm64": "host_arm64", "amd64": "host_x64"}[runtime.GOARCH] | 
 |  | 
 | // NewLocalFuchsiaBuild will create a BaseBuild with path layouts corresponding | 
 | // to a local Fuchsia checkout | 
 | func NewLocalFuchsiaBuild() (Build, error) { | 
 | 	fuchsiaDir := os.Getenv("FUCHSIA_DIR") | 
 | 	if fuchsiaDir == "" { | 
 | 		// Fall back to relative path from this file | 
 | 		fuchsiaDir = filepath.Join("..", "..") | 
 | 	} | 
 |  | 
 | 	fxBuildDir := filepath.Join(fuchsiaDir, ".fx-build-dir") | 
 | 	contents, err := ioutil.ReadFile(fxBuildDir) | 
 | 	if err != nil { | 
 | 		return nil, fmt.Errorf("failed to read %q: %s", fxBuildDir, err) | 
 | 	} | 
 |  | 
 | 	buildDir := strings.TrimSpace(string(contents)) | 
 | 	if !filepath.IsAbs(buildDir) { | 
 | 		buildDir = filepath.Join(fuchsiaDir, buildDir) | 
 | 	} | 
 | 	prebuiltDir := filepath.Join(fuchsiaDir, "prebuilt") | 
 |  | 
 | 	platform, ok := Platforms[runtime.GOOS] | 
 | 	if !ok { | 
 | 		return nil, fmt.Errorf("unsupported os: %s", runtime.GOOS) | 
 | 	} | 
 |  | 
 | 	fxConfig := filepath.Join(buildDir, "fx.config") | 
 | 	file, err := os.Open(fxConfig) | 
 | 	if err != nil { | 
 | 		return nil, fmt.Errorf("failed to open %q: %s", fxConfig, err) | 
 | 	} | 
 | 	defer file.Close() | 
 |  | 
 | 	properties := map[string]string{} | 
 | 	re := regexp.MustCompile(`^([^=]+)=(?:'([^']+)'|(.+))?$`) | 
 | 	scanner := bufio.NewScanner(file) | 
 | 	for scanner.Scan() { | 
 | 		m := re.FindStringSubmatch(scanner.Text()) | 
 | 		if m != nil { | 
 | 			properties[m[1]] = m[2] | 
 | 		} | 
 | 	} | 
 |  | 
 | 	arch, found := properties["FUCHSIA_ARCH"] | 
 | 	if !found { | 
 | 		return nil, fmt.Errorf("no arch in %s", fxConfig) | 
 | 	} | 
 |  | 
 | 	archInfo, ok := Archs[arch] | 
 | 	if !ok { | 
 | 		supported := make([]string, 0, len(Archs)) | 
 | 		for k := range Archs { | 
 | 			supported = append(supported, k) | 
 | 		} | 
 | 		return nil, fmt.Errorf("unsupported arch: %s (supported: %v)", arch, supported) | 
 | 	} | 
 |  | 
 | 	binary := archInfo.Binary | 
 | 	kernel := archInfo.Kernel | 
 | 	platform += "-" + arch | 
 |  | 
 | 	clangDir := filepath.Join(prebuiltDir, "third_party/clang", platform) | 
 | 	qemuDir := filepath.Join(prebuiltDir, "third_party/qemu", platform) | 
 | 	imgDir := filepath.Join(buildDir, "obj", "build", "images", "fuchsia", "fuchsia") | 
 |  | 
 | 	build := &BaseBuild{ | 
 | 		Paths: map[string]string{ | 
 | 			"zbi":             filepath.Join(imgDir, "fuchsia.zbi"), | 
 | 			"fvm":             filepath.Join(buildDir, hostDir, "fvm"), | 
 | 			"zbitool":         filepath.Join(buildDir, hostDir, "zbi"), | 
 | 			"blk":             filepath.Join(imgDir, "fvm.blk"), | 
 | 			"qemu":            filepath.Join(qemuDir, "bin", binary), | 
 | 			"kernel":          filepath.Join(buildDir, kernel), | 
 | 			"llvm-symbolizer": filepath.Join(clangDir, "bin", "llvm-symbolizer"), | 
 | 			"symbolizer":      filepath.Join(buildDir, hostDir, "symbolize"), | 
 | 			"fuzzers.json":    filepath.Join(buildDir, "fuzzers.json"), | 
 | 		}, | 
 | 		IDs: []string{ | 
 | 			filepath.Join(clangDir, "lib", "debug", ".build-id"), | 
 | 			filepath.Join(buildDir, ".build-id"), | 
 | 		}, | 
 | 	} | 
 | 	if err := build.LoadFuzzers(); err != nil { | 
 | 		return nil, err | 
 | 	} | 
 |  | 
 | 	return build, nil | 
 | } | 
 |  | 
 | // Convenience type alias for heterogenous metadata objects in fuzzers.json | 
 | type fuzzerMetadata map[string]string | 
 |  | 
 | // LoadFuzzers reads and parses fuzzers.json to populate the build's map of Fuzzers. | 
 | // Unless an error is returned, any previously loaded fuzzers will be discarded. | 
 | func (b *BaseBuild) LoadFuzzers() error { | 
 | 	paths, err := b.Path("fuzzers.json") | 
 | 	if err != nil { | 
 | 		return err | 
 | 	} | 
 |  | 
 | 	jsonPath := paths[0] | 
 |  | 
 | 	glog.Infof("Loading fuzzers from %q", jsonPath) | 
 |  | 
 | 	jsonBlob, err := ioutil.ReadFile(jsonPath) | 
 | 	if err != nil { | 
 | 		return fmt.Errorf("failed to read %q: %s", jsonPath, err) | 
 | 	} | 
 |  | 
 | 	var metadataList []fuzzerMetadata | 
 | 	if err := json.Unmarshal(jsonBlob, &metadataList); err != nil { | 
 | 		return fmt.Errorf("failed to parse %q: %s", jsonPath, err) | 
 | 	} | 
 |  | 
 | 	// Condense metadata entries by label | 
 | 	metadataByLabel := make(map[string]fuzzerMetadata) | 
 | 	for _, metadata := range metadataList { | 
 | 		label, found := metadata["label"] | 
 | 		if !found { | 
 | 			return fmt.Errorf("failed to parse %q: entry missing label", jsonPath) | 
 | 		} | 
 |  | 
 | 		if _, found := metadataByLabel[label]; !found { | 
 | 			metadataByLabel[label] = make(fuzzerMetadata) | 
 | 		} | 
 |  | 
 | 		for k, v := range metadata { | 
 | 			if v != "" { | 
 | 				metadataByLabel[label][k] = v | 
 | 			} | 
 | 		} | 
 | 	} | 
 |  | 
 | 	b.Fuzzers = make(map[string]*Fuzzer) | 
 | 	for label, metadata := range metadataByLabel { | 
 | 		pkg, found := metadata["package"] | 
 | 		if !found { | 
 | 			return fmt.Errorf("failed to parse %q: no package for %q", jsonPath, label) | 
 | 		} | 
 |  | 
 | 		fuzzer, found := metadata["fuzzer"] | 
 | 		if !found { | 
 | 			return fmt.Errorf("failed to parse %q: no fuzzer for %q", jsonPath, label) | 
 | 		} | 
 |  | 
 | 		f := NewFuzzer(b, pkg, fuzzer) | 
 | 		b.Fuzzers[f.Name] = f | 
 | 	} | 
 |  | 
 | 	return nil | 
 | } | 
 |  | 
 | // ListFuzzers lists the names of fuzzers present in the build, excluding any | 
 | // that we don't want ClusterFuzz to actually pick up. We can't just omit the | 
 | // example fuzzers from the build entirely because they are used in integration | 
 | // testing. | 
 | func (b *BaseBuild) ListFuzzers() []string { | 
 | 	var names []string | 
 | 	for _, fuzzer := range b.Fuzzers { | 
 | 		if !fuzzer.IsExample() { | 
 | 			names = append(names, fuzzer.Name) | 
 | 		} | 
 | 	} | 
 | 	return names | 
 | } | 
 |  | 
 | // Fuzzer finds the Fuzzer with the given name, if available | 
 | func (b *BaseBuild) Fuzzer(name string) (*Fuzzer, error) { | 
 | 	fuzzer, found := b.Fuzzers[name] | 
 | 	if !found { | 
 | 		return nil, fmt.Errorf("no such fuzzer: %s", name) | 
 | 	} | 
 | 	return fuzzer, nil | 
 | } | 
 |  | 
 | // Prepare is a no-op for simple builds | 
 | func (b *BaseBuild) Prepare() error { | 
 | 	return nil | 
 | } | 
 |  | 
 | // Path returns the absolute paths to the list of files indicated by keys. This | 
 | // allows callers to abstract away the detail of where specific file resources | 
 | // are. | 
 | func (b *BaseBuild) Path(keys ...string) ([]string, error) { | 
 | 	paths := make([]string, len(keys)) | 
 | 	for i, key := range keys { | 
 | 		if path, found := b.Paths[key]; found { | 
 | 			paths[i] = path | 
 | 		} else { | 
 | 			return nil, fmt.Errorf("no path for %q", key) | 
 | 		} | 
 | 	} | 
 | 	return paths, nil | 
 | } | 
 |  | 
 | var logPrefixRegex = regexp.MustCompile(`[0-9\[\]\.]*\[klog\] INFO: `) | 
 |  | 
 | // Remove timestamps, etc. | 
 | func stripLogPrefix(line string) string { | 
 | 	return logPrefixRegex.ReplaceAllString(line, "") | 
 | } | 
 |  | 
 | // Symbolize reads from in and replaces symbolizer markup with debug | 
 | // information before writing the result to out.  This is blocking, and does | 
 | // not propagate EOFs from in to out. | 
 | func (b *BaseBuild) Symbolize(in io.ReadCloser, out io.Writer) error { | 
 | 	// Close `in` on return so that the fuzzer doesn't block on a write if an | 
 | 	// early exit occurs later in the output-processing chain. | 
 | 	defer in.Close() | 
 |  | 
 | 	paths, err := b.Path("symbolizer", "llvm-symbolizer") | 
 | 	if err != nil { | 
 | 		return err | 
 | 	} | 
 |  | 
 | 	symbolizer, llvmSymbolizer := paths[0], paths[1] | 
 |  | 
 | 	args := []string{"-llvm-symbolizer", llvmSymbolizer} | 
 | 	for _, dir := range b.IDs { | 
 | 		args = append(args, "-build-id-dir", dir) | 
 | 	} | 
 | 	cmd := NewCommand(symbolizer, args...) | 
 | 	cmd.Stdin = in | 
 | 	pipe, err := cmd.StdoutPipe() | 
 | 	if err != nil { | 
 | 		return err | 
 | 	} | 
 | 	if err := cmd.Start(); err != nil { | 
 | 		return err | 
 | 	} | 
 | 	scanner := bufio.NewScanner(pipe) | 
 |  | 
 | 	for scanner.Scan() { | 
 | 		io.WriteString(out, stripLogPrefix(scanner.Text())+"\n") | 
 | 	} | 
 |  | 
 | 	if err := scanner.Err(); err != nil { | 
 | 		return fmt.Errorf("failed during scan: %s", err) | 
 | 	} | 
 |  | 
 | 	return cmd.Wait() | 
 | } |