blob: 3f6498ee077fe3db887e755b14b60e833c569402 [file] [log] [blame]
// Copyright 2021 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build ignore
// +build ignore
package main
// benchstat-ratio.go prints the relative ratios of similarly named benchstat
// summary lines. The benchstat tool (written in Go) is not specific to Wuffs.
// It is documented at https://godoc.org/golang.org/x/perf/cmd/benchstat
//
// Out of the box, benchstat (when passed two filename arguments) compares
// *identically* named lines (i.e. before and after a code change). This
// program computes ratios for *different-but-similar* named lines (i.e.
// multiple implementations or compilers for the same algorithm or test data),
// grouped by the name fragment after the first '_' and before the first '/'.
//
// By default, the baseline case (normalized to a ratio of 1.00x) is the one
// whose name starts with "mimic_" and contains "/gcc".
//
// For example, given benchstat output like:
//
// wuffs_crc32_ieee_10k/clang9 8.32GB/s ± 4%
// wuffs_crc32_ieee_100k/clang9 10.8GB/s ± 1%
// mimic_crc32_ieee_10k/clang9 771MB/s ± 0%
// mimic_crc32_ieee_100k/clang9 772MB/s ± 0%
// wuffs_crc32_ieee_10k/gcc10 8.68GB/s ± 0%
// wuffs_crc32_ieee_100k/gcc10 10.9GB/s ± 0%
// mimic_crc32_ieee_10k/gcc10 775MB/s ± 0%
// mimic_crc32_ieee_100k/gcc10 773MB/s ± 0%
//
// Piping that through this program prints:
//
// wuffs_crc32_ieee_10k/clang9 8.32GB/s ± 4% 10.74x
// wuffs_crc32_ieee_100k/clang9 10.8GB/s ± 1% 13.97x
// mimic_crc32_ieee_10k/clang9 771MB/s ± 0% 0.99x
// mimic_crc32_ieee_100k/clang9 772MB/s ± 0% 1.00x
// wuffs_crc32_ieee_10k/gcc10 8.68GB/s ± 0% 11.20x
// wuffs_crc32_ieee_100k/gcc10 10.9GB/s ± 0% 14.10x
// mimic_crc32_ieee_10k/gcc10 775MB/s ± 0% 1.00x
// mimic_crc32_ieee_100k/gcc10 773MB/s ± 0% 1.00x
//
// There are two groups here: "crc32_ieee_10k" and "crc32_ieee_100k".
//
// Usage: benchstat etc | go run benchstat-ratio.go
import (
"bufio"
"flag"
"fmt"
"os"
"strconv"
"strings"
)
var (
baseline = flag.String("baseline", "mimic", "benchmark name prefix of the 1.00x case")
)
func main() {
if err := main1(); err != nil {
os.Stderr.WriteString(err.Error() + "\n")
os.Exit(1)
}
}
func main1() error {
flag.Parse()
baselineUnderscore = *baseline + "_"
r := bufio.NewScanner(os.Stdin)
for r.Scan() {
line := strings.TrimSpace(string(r.Bytes()))
pmIndex := strings.Index(line, plusMinus)
if pmIndex < 0 {
// Collapse multiple blank lines to a single one.
if (len(entries) > 0) && (entries[len(entries)-1].key != "") {
entries = append(entries, entry{})
}
continue
}
// Find the "/gcc10" in "mimic_etc/gcc10".
if (gccSuffix == "") && strings.HasPrefix(line, baselineUnderscore) {
if i := strings.Index(line, slashGCC); i >= 0 {
gccSuffix = line[i:]
if j := strings.IndexByte(gccSuffix, ' '); j >= 0 {
gccSuffix = gccSuffix[:j]
}
}
}
key, val := parse(line[:pmIndex])
if key == "" {
return fmt.Errorf("could not parse %q", line)
} else if _, ok := values[key]; ok {
return fmt.Errorf("duplicate key for %q", line)
}
entries = append(entries, entry{key, line})
values[key] = val
}
if err := r.Err(); err != nil {
return err
}
for _, e := range entries {
if e.key == "" {
fmt.Println()
continue
}
eVal, eOK := values[e.key]
bVal, bOK := values[baselineKeyFor(e.key)]
if eOK && bOK {
switch e.key[0] {
case '1':
fmt.Printf("%s %6.2fx\n", e.line, bVal/eVal)
continue
case '2':
fmt.Printf("%s %6.2fx\n", e.line, eVal/bVal)
continue
}
}
fmt.Printf("%s\n", e.line)
}
return nil
}
func parse(line string) (key string, val float64) {
i := strings.IndexByte(line, ' ')
if i < 0 {
return "", 0
}
k, v := line[:i], strings.TrimSpace(line[i+1:])
prefix, multiplier := "", 1e0
switch {
case strings.HasSuffix(v, suffix_GBs):
prefix, multiplier = prefix2, 1e9
v = v[:len(v)-len(suffix_GBs)]
case strings.HasSuffix(v, suffix_MBs):
prefix, multiplier = prefix2, 1e6
v = v[:len(v)-len(suffix_MBs)]
case strings.HasSuffix(v, suffix_KBs):
prefix, multiplier = prefix2, 1e3
v = v[:len(v)-len(suffix_KBs)]
case strings.HasSuffix(v, suffix__Bs):
prefix, multiplier = prefix2, 1e0
v = v[:len(v)-len(suffix__Bs)]
case strings.HasSuffix(v, suffix_ns):
prefix, multiplier = prefix1, 1e0
v = v[:len(v)-len(suffix_ns)]
case strings.HasSuffix(v, suffix_µs):
prefix, multiplier = prefix1, 1e3
v = v[:len(v)-len(suffix_µs)]
case strings.HasSuffix(v, suffix_ms):
prefix, multiplier = prefix1, 1e6
v = v[:len(v)-len(suffix_ms)]
case strings.HasSuffix(v, suffix__s):
prefix, multiplier = prefix1, 1e9
v = v[:len(v)-len(suffix__s)]
default:
return "", 0
}
x, err := strconv.ParseFloat(v, 64)
if err != nil {
return "", 0
}
return prefix + string(k), x * multiplier
}
func baselineKeyFor(key string) (bKey string) {
if key == "" {
return ""
}
i := strings.IndexByte(key, '_')
if i < 0 {
return ""
}
j := strings.IndexByte(key[i:], '/')
if j < 0 {
j = len(key) - i
}
return key[:1] + *baseline + key[i:i+j] + gccSuffix
}
type entry struct {
key string
line string
}
var (
entries []entry
values = map[string]float64{}
baselineUnderscore string
gccSuffix string
)
const (
plusMinus = " ± "
slashGCC = "/gcc"
prefix1 = "1" // val is nanoseconds.
prefix2 = "2" // val is bytes per second.
suffix_GBs = "GB/s"
suffix_MBs = "MB/s"
suffix_KBs = "KB/s"
suffix__Bs = "B/s"
suffix_ns = "ns"
suffix_µs = "µs"
suffix_ms = "ms"
suffix__s = "s"
)