[tools][fuzz] Use tmpfs for faster mutable data storage
Particularly when writing large corpora consisting of many small files,
minfs was acting as a performance bottleneck. This changes undercoat to
transparently map data/ paths to tmp/ paths behind the scenes.
With this change, the e2e test shows a 10x speedup when transferring a
2k element corpus.
Bug: 92637
Change-Id: Ief50ea60be24f5b9024b84e5354f44a0c87830ba
Reviewed-on: https://fuchsia-review.googlesource.com/c/fuchsia/+/642353
Reviewed-by: Aaron Green <aarongreen@google.com>
Commit-Queue: Cameron Finucane <eep@google.com>
diff --git a/tools/fuzz/command_mock.go b/tools/fuzz/command_mock.go
index 0550541..37cea6a 100644
--- a/tools/fuzz/command_mock.go
+++ b/tools/fuzz/command_mock.go
@@ -35,12 +35,21 @@
}
fuzzerName := fmt.Sprintf("%s/%s", m[1], m[2])
- // Look up the artifact prefix to use
+ // Look up arguments that we want to test
var artifactPrefix string
+ var mergeFile string
+ var corpusPath string
for _, arg := range c.args[1:] {
+ // Save first non-option arg
+ if corpusPath == "" && !strings.HasPrefix(arg, "-") {
+ corpusPath = arg
+ continue
+ }
+
if parts := strings.Split(arg, "="); parts[0] == "-artifact_prefix" {
artifactPrefix = parts[1]
- break
+ } else if parts[0] == "-merge_control_file" {
+ mergeFile = parts[1]
}
}
if artifactPrefix == "" {
@@ -49,6 +58,11 @@
artifactLine := fmt.Sprintf("artifact_prefix='%s'; "+
"Test unit written to %scrash-1312", artifactPrefix, artifactPrefix)
+ if corpusPath == "" {
+ return nil, fmt.Errorf("run command missing output corpus dir: %q", c.args)
+ }
+ corpusLine := fmt.Sprintf("INFO: 4 files found in %s", corpusPath)
+
var output []string
switch fuzzerName {
case "foo/bar":
@@ -60,11 +74,20 @@
}
output = append(filler,
fmt.Sprintf("running %v", c.args),
+ corpusLine,
"==123==", // pid
"MS: ", // mut
"Deadly signal",
artifactLine,
)
+
+ // This wouldn't actually be emitted during a non-merge run, but we
+ // want to exercise an option with a path
+ if mergeFile != "" {
+ output = append(output,
+ fmt.Sprintf("MERGE-INNER: using the control file '%s'", mergeFile),
+ )
+ }
case "fail/nopid":
// No PID
output = []string{
diff --git a/tools/fuzz/fuzzer.go b/tools/fuzz/fuzzer.go
index 1fb9bde..1f5c55c 100644
--- a/tools/fuzz/fuzzer.go
+++ b/tools/fuzz/fuzzer.go
@@ -52,16 +52,35 @@
}
}
+// Map paths as referenced by ClusterFuzz to internally-used paths as seen by
+// libFuzzer, SFTP, etc.
+func translatePath(relpath string) string {
+ // Note: we can't use path.Join or other path functions that normalize the path
+ // because it will drop trailing slashes, which is important to preserve in
+ // places like artifact_prefix.
+
+ // Rewrite all references to data/ to tmp/ for better performance
+ if strings.HasPrefix(relpath, "data/") {
+ relpath = "tmp/" + strings.TrimPrefix(relpath, "data/")
+ }
+
+ return relpath
+}
+
// AbsPath returns the absolute target path for a given relative path in a
// fuzzer package. The path may differ depending on whether it is identified as
// a resource, data, or neither.
func (f *Fuzzer) AbsPath(relpath string) string {
+ relpath = translatePath(relpath)
+
if strings.HasPrefix(relpath, "/") {
return relpath
} else if strings.HasPrefix(relpath, "pkg/") {
return fmt.Sprintf("/pkgfs/packages/%s/0/%s", f.pkg, relpath[4:])
} else if strings.HasPrefix(relpath, "data/") {
return fmt.Sprintf("/data/r/sys/fuchsia.com:%s:0#meta:%s/%s", f.pkg, f.cmx, relpath[5:])
+ } else if strings.HasPrefix(relpath, "tmp/") {
+ return fmt.Sprintf("/tmp/r/sys/fuchsia.com:%s:0#meta:%s/%s", f.pkg, f.cmx, relpath[4:])
} else {
return fmt.Sprintf("/%s", relpath)
}
@@ -75,9 +94,9 @@
for _, arg := range args {
submatch := re.FindStringSubmatch(arg)
if submatch == nil {
- f.args = append(f.args, arg)
+ f.args = append(f.args, translatePath(arg))
} else {
- f.options[submatch[1]] = submatch[2]
+ f.options[submatch[1]] = translatePath(submatch[2])
}
}
}
@@ -230,7 +249,7 @@
}
// Clear any persistent data in the fuzzer's namespace, resetting its state
- dataPath := f.AbsPath("data/*")
+ dataPath := f.AbsPath("tmp/*")
if err := conn.Command("rm", "-rf", dataPath).Run(); err != nil {
return fmt.Errorf("error clear fuzzer data namespace %q: %s", dataPath, err)
}
@@ -250,11 +269,11 @@
// Ensure artifact_prefix will be writable, and fall back to default if not
// specified
if artPrefix, ok := f.options["artifact_prefix"]; ok {
- if !strings.HasPrefix(strings.TrimLeft(artPrefix, "/"), "data/") {
- return nil, fmt.Errorf("artifact_prefix not in data/ namespace: %q", artPrefix)
+ if !strings.HasPrefix(artPrefix, "tmp/") {
+ return nil, fmt.Errorf("artifact_prefix not in mutable namespace: %q", artPrefix)
}
} else {
- f.options["artifact_prefix"] = "data/"
+ f.options["artifact_prefix"] = "tmp/"
}
// The overall flow of fuzzer output data is as follows:
diff --git a/tools/fuzz/fuzzer_test.go b/tools/fuzz/fuzzer_test.go
index f38956b..856261d 100644
--- a/tools/fuzz/fuzzer_test.go
+++ b/tools/fuzz/fuzzer_test.go
@@ -21,7 +21,7 @@
absPaths := map[string]string{
"pkg/data/relpath": "/pkgfs/packages/foo/0/data/relpath",
"/pkg/data/relpath": "/pkg/data/relpath",
- "data/relpath": "/data/r/sys/fuchsia.com:foo:0#meta:bar.cmx/relpath",
+ "data/relpath": "/tmp/r/sys/fuchsia.com:foo:0#meta:bar.cmx/relpath",
"/data/relpath": "/data/relpath",
"relpath": "/relpath",
"/relpath": "/relpath",
@@ -94,7 +94,7 @@
t.Fatalf("failed to load fuzzer: %s", err)
}
- f.Parse(args)
+ f.Parse(append(args, "data/corpus"))
var outBuf bytes.Buffer
artifacts, err := f.Run(conn, &outBuf, "/some/artifactDir")
@@ -103,7 +103,8 @@
}
func TestRun(t *testing.T) {
- out, artifacts, err := runFuzzer(t, "foo/bar", nil, FuzzerNormal)
+ out, artifacts, err := runFuzzer(t, "foo/bar",
+ []string{"-merge_control_file=data/.mergefile"}, FuzzerNormal)
if err != nil {
t.Fatalf("failed to run fuzzer: %s", err)
}
@@ -119,7 +120,7 @@
}
// Check for artifact detection
- artifactAbsPath := "/data/r/sys/fuchsia.com:foo:0#meta:bar.cmx/crash-1312"
+ artifactAbsPath := "/tmp/r/sys/fuchsia.com:foo:0#meta:bar.cmx/crash-1312"
if !reflect.DeepEqual(artifacts, []string{artifactAbsPath}) {
t.Fatalf("unexpected artifact list: %s", artifacts)
}
@@ -128,12 +129,21 @@
if !strings.Contains(out, "/some/artifactDir/crash-1312") {
t.Fatalf("artifact prefix not rewritten: %q", out)
}
+
+ // Check that paths in libFuzzer options/args are translated
+ if !strings.Contains(out, "tmp/.mergefile") {
+ t.Fatalf("mergefile prefix not rewritten: %q", out)
+ }
+
+ if !strings.Contains(out, "tmp/corpus") {
+ t.Fatalf("corpus prefix not rewritten: %q", out)
+ }
}
func TestRunWithInvalidArtifactPrefix(t *testing.T) {
args := []string{"-artifact_prefix=foo/bar"}
_, _, err := runFuzzer(t, "foo/bar", args, FuzzerNormal)
- if err == nil || !strings.Contains(err.Error(), "artifact_prefix not in data/") {
+ if err == nil || !strings.Contains(err.Error(), "artifact_prefix not in mutable") {
t.Fatalf("expected failure to run but got: %s", err)
}
}
diff --git a/tools/fuzz/instance_test.go b/tools/fuzz/instance_test.go
index 38622f3..bcd1962 100644
--- a/tools/fuzz/instance_test.go
+++ b/tools/fuzz/instance_test.go
@@ -200,7 +200,7 @@
t.Fatalf("Error getting from instance: %s", err)
}
- expected := []string{"/data/r/sys/fuchsia.com:foo:0#meta:bar.cmx/path/to/remoteFile"}
+ expected := []string{"/tmp/r/sys/fuchsia.com:foo:0#meta:bar.cmx/path/to/remoteFile"}
got := i.Connector.(*mockConnector).PathsGot
if !reflect.DeepEqual(got, expected) {
t.Fatalf("incorrect file get list: %v", got)
@@ -211,7 +211,7 @@
t.Fatalf("Error putting to instance: %s", err)
}
- expected = []string{"/data/r/sys/fuchsia.com:foo:0#meta:bar.cmx/path/to/otherFile"}
+ expected = []string{"/tmp/r/sys/fuchsia.com:foo:0#meta:bar.cmx/path/to/otherFile"}
got = i.Connector.(*mockConnector).PathsPut
if !reflect.DeepEqual(got, expected) {
t.Fatalf("incorrect file put list: %v", got)
@@ -246,7 +246,8 @@
hostArtifactDir := "/art/dir"
var outBuf bytes.Buffer
- if err := i.RunFuzzer(&outBuf, "foo/bar", hostArtifactDir, "-artifact_prefix=data/wow/x"); err != nil {
+ args := []string{"-artifact_prefix=data/wow/x", "data/corpus"}
+ if err := i.RunFuzzer(&outBuf, "foo/bar", hostArtifactDir, args...); err != nil {
t.Fatalf("Error running fuzzer: %s", err)
}
@@ -255,7 +256,7 @@
t.Fatalf("fuzzer output missing host artifact path: %q", out)
}
- expected := []string{"/data/r/sys/fuchsia.com:foo:0#meta:bar.cmx/wow/xcrash-1312"}
+ expected := []string{"/tmp/r/sys/fuchsia.com:foo:0#meta:bar.cmx/wow/xcrash-1312"}
got := i.Connector.(*mockConnector).PathsGot
if !reflect.DeepEqual(got, expected) {
t.Fatalf("incorrect file get list: %v", got)
