internal/engine/bench_test.go

// Copyright 2023 The Shac Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package engine

import (
	"context"
	"errors"
	"path/filepath"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"
)

func BenchmarkPrint_Raw(b *testing.B) {
	root := b.TempDir()
	copyFile(b, root, "testdata/bench/print.star")
	want := "[//print.star:16] running\n"
	benchStarlarkPrint(b, root, "print.star", true, want)
}

func BenchmarkPrint_Git(b *testing.B) {
	root := makeGit(b)
	copyFile(b, root, "testdata/bench/print.star")
	want := "[//print.star:16] running\n"
	benchStarlarkPrint(b, root, "print.star", true, want)
}

func BenchmarkPrint100_Raw(b *testing.B) {
	root := b.TempDir()
	copyFile(b, root, "testdata/bench/print100.star")
	want := strings.Repeat("[//print100.star:16] running\n", 100)
	benchStarlarkPrint(b, root, "print100.star", true, want)
}

func BenchmarkPrint100_Git(b *testing.B) {
	root := makeGit(b)
	copyFile(b, root, "testdata/bench/print100.star")
	want := strings.Repeat("[//print100.star:16] running\n", 100)
	benchStarlarkPrint(b, root, "print100.star", true, want)
}

func BenchmarkCtxEmitFinding(b *testing.B) {
	// Use ctx-emit-finding-warning.star since it only emit once, which makes
	// understanding memory allocation easier.
	root := b.TempDir()
	copyFile(b, root, "testdata/bench/ctx-emit-finding.star")
	copyFile(b, root, "testdata/bench/file.txt")
	r := reportEmitNoPrint{reportNoPrint: reportNoPrint{t: b}}
	o := Options{Report: &r, Dir: root, main: "ctx-emit-finding.star"}
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := Run(context.Background(), &o); err != nil {
			b.Fatal(err)
		}
		r.findings = r.findings[:0]
		r.artifacts = r.artifacts[:0]
	}
}

func BenchmarkCtxEmitArtifact(b *testing.B) {
	root := b.TempDir()
	copyFile(b, root, "testdata/bench/ctx-emit-artifact.star")
	copyFile(b, root, "testdata/bench/file.txt")
	r := reportEmitNoPrint{reportNoPrint: reportNoPrint{t: b}}
	o := Options{Report: &r, Dir: root, main: "ctx-emit-artifact.star"}
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := Run(context.Background(), &o); err != nil {
			b.Fatal(err)
		}
		r.findings = r.findings[:0]
		r.artifacts = r.artifacts[:0]
	}
}

func BenchmarkCtxOsExec(b *testing.B) {
	root := b.TempDir()
	copyFile(b, root, "testdata/bench/ctx-os-exec.star")
	copyFile(b, root, "testdata/bench/stdio.bat")
	copyFile(b, root, "testdata/bench/stdio.sh")
	want := "[//ctx-os-exec.star:21] retcode: 0\n" +
		"[//ctx-os-exec.star:22] stdout: hello from stdout\n" +
		"[//ctx-os-exec.star:23] stderr: hello from stderr\n"
	benchStarlarkPrint(b, root, "ctx-os-exec.star", true, want)
}

func BenchmarkCtxOsExec100(b *testing.B) {
	root := b.TempDir()
	copyFile(b, root, "testdata/bench/ctx-os-exec100.star")
	copyFile(b, root, "testdata/bench/stdio.bat")
	copyFile(b, root, "testdata/bench/stdio.sh")
	want := "[//ctx-os-exec100.star:25] retcode: 0\n" +
		"stdout: hello from stdout\n" +
		"stderr: hello from stderr\n"
	want = strings.Repeat(want, 100)
	benchStarlarkPrint(b, root, "ctx-os-exec100.star", true, want)
}

func BenchmarkCtxScmNewLines_Git(b *testing.B) {
	root := makeGit(b)
	copyBenchSCM(b, root)
	runGit(b, root, "add", "ctx-scm-*.star")
	want := "[//ctx-scm-affected_files-new_lines.star:24] ctx-scm-affected_files-new_lines.star\n" +
		"1: # Copyright 2023 The Shac Authors\n"
	benchStarlarkPrint(b, root, "ctx-scm-affected_files-new_lines.star", false, want)
}

func BenchmarkCtxScmNewLines100_Git(b *testing.B) {
	root := makeGit(b)
	copyBenchSCM(b, root)
	runGit(b, root, "add", "ctx-scm-*.star")
	want := "[//ctx-scm-affected_files-new_lines100.star:24] ctx-scm-affected_files-new_lines.star\n" +
		"1: # Copyright 2023 The Shac Authors\n"
	want = strings.Repeat(want, 100)
	benchStarlarkPrint(b, root, "ctx-scm-affected_files-new_lines100.star", false, want)
}

func BenchmarkCtxScmNewLines_Raw(b *testing.B) {
	root := b.TempDir()
	writeFile(b, root, "a.txt", "First file")
	copyBenchSCM(b, root)
	want := "[//ctx-scm-affected_files-new_lines.star:24] a.txt\n" +
		"1: First file\n"
	benchStarlarkPrint(b, root, "ctx-scm-affected_files-new_lines.star", false, want)
}

// TODO(maruel): Add large synthetic benchmark.

// benchStarlarkPrint benchmarks a starlark file that calls print().
func benchStarlarkPrint(b *testing.B, root, name string, all bool, want string) {
	r := reportPrint{reportNoPrint: reportNoPrint{t: b}}
	o := Options{Report: &r, Dir: root, AllFiles: all, main: name}
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := Run(context.Background(), &o); err != nil {
			var err2 BacktraceableError
			if errors.As(err, &err2) {
				b.Fatal(err2.Backtrace())
			}
			b.Fatal(err)
		}
		if i == 0 {
			if got := r.b.String(); got != want {
				b.Helper()
				b.Fatalf("mismatch (-want +got):\n%s", cmp.Diff(want, got))
			}
		}
		// We need to reset the buffer to reuse it, but Reset() simply ditch the
		// buffer. Use Grow() right after so there's only one memory allocation (as
		// overhead) per test case.
		l := r.b.Len()
		r.b.Reset()
		r.b.Grow(l)
	}
}

func copyBenchSCM(t testing.TB, dst string) {
	m, err := filepath.Glob(filepath.Join("testdata", "bench", "ctx-scm-*.star"))
	if err != nil {
		t.Fatal(err)
	}
	for _, src := range m {
		copyFile(t, dst, src)
	}
}