tests/bench/Makefile - third_party/github.com/leanprover/lean4 - Git at Google


            
            ## CONFIG


            LEAN_BENCHES = binarytrees binarytrees.st deriv const_fold parser qsort rbmap rbmap_10 rbmap_2 rbmap_1 unionfind

            
            CROSS_BENCHES = binarytrees binarytrees.st deriv const_fold rbmap rbmap_10 rbmap_2 rbmap_1


            LEAN_CATS = .lean .no_reuse.lean .no_borrow.lean .no_st.lean

            
            CROSS_CATS = .lean .gc.lean .hs .gc.hs .ml .gc.ml .mlton .gc.mlton .mlkit .gc.mlkit .swift .gc.swift

            
            CROSS_TIME_CATS = .lean .hs .ml .mlton .mlkit .swift

            
            RETIRED_CATS = .gcc.lean .llvm.hs .strict.hs .flambda.ml .lean.perf .hs.perf .ml.perf .mlton.perf .mlkit.perf .swift.perf


            LEAN_FLAGS =

            
            LEANC_FLAGS = -O3 -DNDEBUG

            
            GHC_FLAGS = -O3

            
            OCAML_FLAGS = -O3

            
            MLTON_FLAGS =

            
            MLKIT_FLAGS =

            
            SWIFTC_FLAGS = -O -whole-module-optimization


            TEMCI_FLAGS = --settings cross.yaml


            TEMCI ?= temci

            
            LEAN_BIN ?= ../../build/release/stage1/bin

            
            GHC ?= ghc

            
            OCAML ?= ocamlopt.opt

            
            MLTON_BIN ?= /usr/bin

            
            MLKIT ?= mlkit

            
            SWIFTC ?= swiftc


            PARSER_TEST_FILE ?= ../../src/Init/Prelude.lean


            ## IMPLEMENTATION


            LEAN_INPUTS = $(foreach bench,$(LEAN_BENCHES), $(foreach cat, $(LEAN_CATS), $(bench)$(cat)))

            
            CROSS_INPUTS = $(foreach bench,$(CROSS_BENCHES), $(foreach cat, $(CROSS_CATS), $(bench)$(cat)))


            .SECONDARY: $(LEAN_INPUTS:%=%.out) $(LEAN_INPUTS:%=bench/%.bench) $(CROSS_INPUTS:%=%.out) $(CROSS_INPUTS:%=bench/%.bench)

            
            .DELETE_ON_ERROR:


            all: report_lean.tex report_cross.tex report


            # disable some built-in rules

            
            %.lean:

            
            %.out: %


            %.lean.c: %.lean

            
            	$(LEAN_BIN)/lean --c=$@ $(LEAN_FLAGS) $<

            
            %.lean.out: %.lean.c

            
            	$(LEAN_BIN)/leanc $(LEANC_FLAGS) -o $@ $<

            
            # Binaries x.lean.out and x.gcc.lean.out etc. are produced by the

            
            # same rules and x.lean source file by copying the latter to

            
            # x.gcc.lean. This also avoids conflicts between intermediate

            
            # files of the two binaries.

            
            %.gcc.lean.out: LEAN_BIN = $(LEAN_GCC_BIN)

            
            %.gcc.lean: %.lean; ln -f $< $@

            
            %.no_reuse.lean.c: LEAN_BIN = $(LEAN_NO_REUSE_BIN)

            
            %.no_reuse.lean.out: LEAN_BIN = $(LEAN_NO_REUSE_BIN)

            
            %.no_reuse.lean: %.lean; ln -f $< $@

            
            %.no_borrow.lean.c: LEAN_BIN = $(LEAN_NO_BORROW_BIN)

            
            %.no_borrow.lean.out: LEAN_BIN = $(LEAN_NO_BORROW_BIN)

            
            %.no_borrow.lean: %.lean; ln -f $< $@

            
            %.no_st.lean.out: LEAN_BIN = $(LEAN_NO_ST_BIN)

            
            %.no_st.lean: %.lean; ln -f $< $@


            %.hs.out: %.hs

            
            	$(GHC) $(GHC_FLAGS) -rtsopts $< -o $@

            
            %.llvm.hs.out: GHC_FLAGS += -fllvm

            
            %.llvm.hs: %.hs; ln -f $< $@

            
            %.strict.hs.out: GHC_FLAGS += -XStrict

            
            %.strict.hs: %.hs; ln -f $< $@


            binarytrees.hs: binarytrees.ghc-6.hs; ln -f $< $@

            
            # NOTE: changed `-N4` rtsopt to `-N` to be less system-dependent

            
            binarytrees%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -threaded -rtsopts -with-rtsopts='-N -K128M -H'

            
            binarytrees.st%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -rtsopts -with-rtsopts='-K128M -H'


            %.ml.out: %.ml

            
            	$(OCAML) $(OCAML_FLAGS) $< -o $@

            
            %.flambda.ml.out: OCAML = $(OCAML_FLAMBDA)

            
            %.flambda.ml: %.ml; ln -f $< $@


            binarytrees.ml: binarytrees5_multicore.ml; ln -f $< $@

            
            binarytrees.st.ml: binarytrees5.ml; ln -f $< $@

            
            binarytrees%ml.out: OCAML_FLAGS += -noassert -fPIC -nodynlink -inline 100 -O3 -package domainslib -linkpkg

            
            binarytrees%ml.out: OCAML = ocamlfind ocamlopt


            binarytrees.st.sml: binarytrees.st.mlton-2.sml; ln -f $< $@

            
            %.mlton.out: %.sml

            
            	$(MLTON_BIN)/mlton $(MLTON_FLAGS) -output $@ $<

            
            %.gc.mlton.out: MLTON_FLAGS = -profile time

            
            %.gc.sml: %.sml; ln -f $< $@


            %.mlkit.out: %.sml

            
            	$(MLKIT) $(MLKIT_FLAGS) -o $@ $<


            %.swift.out: %.swift

            
            	$(SWIFTC) $(SWIFTC_FLAGS) -o $@ $<


            binarytrees%swift.out: SWIFTC_FLAGS = -Ounchecked -I Include/swift/apr


            bench:

            
            	-@mkdir bench


            bench/%.bench: %.out | bench

            
            	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS) -d $< "./$< $(BENCH_PARAMS)" --out $@


            bench/parser.%.bench: BENCH_PARAMS = $(PARSER_TEST_FILE) 50


            bench/binarytrees.%.bench: BENCH_PARAMS = 21

            
            bench/binarytrees.ml.bench: BENCH_PARAMS = $$(nproc) 21

            
            bench/binarytrees.gc.ml.bench: BENCH_PARAMS = $$(nproc) 21


            bench/deriv.%.bench: BENCH_PARAMS = 10


            bench/const_fold.%.bench: BENCH_PARAMS = 23


            bench/qsort.%.bench: BENCH_PARAMS = 400


            bench/rbmap.%.bench: BENCH_PARAMS = 2000000


            rbmap_1.%.out: rbmap_checkpoint.%.out; ln -f $< $@

            
            bench/rbmap_1.%.bench: BENCH_PARAMS = 2000000 1


            rbmap_2.%.out: rbmap_checkpoint.%.out; ln -f $< $@

            
            bench/rbmap_2.%.bench: BENCH_PARAMS = 2000000 2


            rbmap_10.%.out: rbmap_checkpoint.%.out; ln -f $< $@

            
            bench/rbmap_10.%.bench: BENCH_PARAMS = 2000000 10


            bench/unionfind.%.bench: BENCH_PARAMS = 3000000


            bench/%gc.lean.bench: %lean.out | bench

            
            	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\

            
                -d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio -S 'lean_inc_ref_cold,lean_inc_ref_n_cold,lean_dec_ref_cold,lean_del_core,lean_free_small' | ./lean-gc.py"\

            
            	  --runner output --out $@


            bench/%gc.hs.bench: %hs.out | bench

            
            	$(TEMCI) short exec $(TEMCI_FLAGS)\

            
            		-d $< "./$< +RTS -t --machine-readable -RTS $(BENCH_PARAMS) 2>&1 >/dev/null | ./ghc-gc.py"\

            
            		--runner output --out $@


            bench/%gc.ml.bench: %ml.out | bench

            
            	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\

            
            		-d $< "time -ao $@.tmp -f '%U' olly trace $@.tmp './$< $(BENCH_PARAMS)' > /dev/null && ./ocaml-gc.py < $@.tmp"\

            
            		--runner output --out $@


            bench/%gc.mlton.bench: %gc.mlton.out | bench

            
            	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\

            
            		-d $< './$< $(BENCH_PARAMS) >/dev/null && $(MLTON_BIN)/mlprof ./$< mlmon.out | awk '\''$$1 == "<gc>" { print "gc: " substr($$2, 0, length($$2)-1); found = 1 } END { if (found != 1) { print "gc: 0.000001" } }'\'\

            
            		--runner output --out $@


            bench/%gc.mlkit.bench: %gc.mlkit.out | bench

            
            	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\

            
            		-d $< "time -f '%e' ./$< -report_gc $(BENCH_PARAMS) 2>&1 >/dev/null | ./mlkit-gc.py"\

            
            		--runner output --out $@


            bench/%gc.swift.bench: %swift.out | bench

            
            	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\

            
                -d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio | grep -E 'retain|release|dealloc|free' | ./lean-gc.py"\

            
            	  --runner output --out $@


            bench/%.perf.bench: %.out | bench

            
            	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\

            
            		-d $< "echo > $@.tmp && time -af '%e' perf stat -e cache-misses -x ';' ./$< $(BENCH_PARAMS) 2>&1 >/dev/null | ./perf.py"\

            
            		--runner output --out $@


            # no benchmarksgame versions

            
            bench/binarytrees.mlton.bench: ; touch $@

            
            bench/binarytrees.gc.mlton.bench: ; touch $@

            
            bench/binarytrees.mlton.perf.bench: ; touch $@

            
            bench/binarytrees.mlkit.bench: ; touch $@

            
            bench/binarytrees.gc.mlkit.bench: ; touch $@

            
            bench/binarytrees.mlkit.perf.bench: ; touch $@


            bench_lean: $(LEAN_INPUTS:%=bench/%.bench)

            
            bench_cross: $(CROSS_INPUTS:%=bench/%.bench)


            report/lean: ; -@mkdir -p $@

            
            report/cross: ; -@mkdir -p $@


            report/lean/%: $(foreach cat, $(LEAN_CATS), bench/%$(cat).bench) | report/lean

            
            	cat $^ > $@.tmp

            
            	$(TEMCI) report $@.tmp --settings cross.yaml --reporter html2 --html2_out $@ --html2_force_override --properties etime


            report/cross/%: $(foreach cat, $(CROSS_TIME_CATS), bench/%$(cat).bench) | report/cross

            
            	cat $^ > $@.tmp

            
            	$(TEMCI) report $@.tmp --settings cross.yaml --reporter html2 --html2_out $@ --html2_force_override --properties etime


            report/index.html: $(LEAN_BENCHES:%=report/lean/%) $(CROSS_BENCHES:%=report/cross/%)

            
            	echo "<html><h1>Lean variant benchmarks</h1><ul>"> $@

            
            	for bench in $(LEAN_BENCHES); do \

            
            		echo "<li><a href='lean/$$bench/report.html'>$$bench</a><br><img src='lean/$$bench/fig___SinglesProperty______etime___25.0.svg'></img></li>" >> $@; \

            
            	done

            
            	echo "</ul><h1>Cross-language benchmarks</h1><ul>">> $@

            
            	for bench in $(CROSS_BENCHES); do \

            
            		echo "<li><a href='cross/$$bench/report.html'>$$bench</a><br><img src='cross/$$bench/fig___SinglesProperty______etime___25.0.svg'></img></li>" >> $@; \

            
            	done

            
            	echo "</ul></html>">> $@


            report: report/index.html


            # yes.

            
            space = $() $()


            report_lean.csv report_lean_rbmap.csv: bench_lean

            
            	BENCHES=$(subst $(space),:,$(LEAN_BENCHES)) CATS=$(subst $(space),:,$(LEAN_CATS)) ./report.py report_lean

            
            	column -s';' -t < $@


            report_cross.csv report_cross_rbmap.csv: bench_cross

            
            	BENCHES=$(subst $(space),:,$(CROSS_BENCHES)) CATS=$(subst $(space),:,$(CROSS_CATS)) ./report.py report_cross

            
            	column -s';' -t < $@


            TO_TEX = 's/-/---/g;s/%/\\%/g;s/\.$$//g;s/(\([0-9]\)\([0-9]\))/\\ensuremath{\\tilde{\1}\\tilde{\2}}/g;s/(\([0-9]\))/\\ensuremath{\\tilde{\1}}/g;s/;/ \& /g;s/$$/\\\\/'

            
            report_%.tex: report_%.csv

            
            	tail -n +2 $< | head -n -1 | sed 's/^\([^;]\+\)/\\verb!\1!/;'$(TO_TEX) > $@

            
            	echo -n '\midrule ' >> $@

            
            	tail -n 1 $< | sed 's/. /.\\ /;'$(TO_TEX) >> $@


            clean:

            
            	-rm *.out bench/*
	## CONFIG

	LEAN_BENCHES = binarytrees binarytrees.st deriv const_fold parser qsort rbmap rbmap_10 rbmap_2 rbmap_1 unionfind
	CROSS_BENCHES = binarytrees binarytrees.st deriv const_fold rbmap rbmap_10 rbmap_2 rbmap_1

	LEAN_CATS = .lean .no_reuse.lean .no_borrow.lean .no_st.lean
	CROSS_CATS = .lean .gc.lean .hs .gc.hs .ml .gc.ml .mlton .gc.mlton .mlkit .gc.mlkit .swift .gc.swift
	CROSS_TIME_CATS = .lean .hs .ml .mlton .mlkit .swift
	RETIRED_CATS = .gcc.lean .llvm.hs .strict.hs .flambda.ml .lean.perf .hs.perf .ml.perf .mlton.perf .mlkit.perf .swift.perf

	LEAN_FLAGS =
	LEANC_FLAGS = -O3 -DNDEBUG
	GHC_FLAGS = -O3
	OCAML_FLAGS = -O3
	MLTON_FLAGS =
	MLKIT_FLAGS =
	SWIFTC_FLAGS = -O -whole-module-optimization

	TEMCI_FLAGS = --settings cross.yaml

	TEMCI ?= temci
	LEAN_BIN ?= ../../build/release/stage1/bin
	GHC ?= ghc
	OCAML ?= ocamlopt.opt
	MLTON_BIN ?= /usr/bin
	MLKIT ?= mlkit
	SWIFTC ?= swiftc

	PARSER_TEST_FILE ?= ../../src/Init/Prelude.lean

	## IMPLEMENTATION

	LEAN_INPUTS = $(foreach bench,$(LEAN_BENCHES), $(foreach cat, $(LEAN_CATS), $(bench)$(cat)))
	CROSS_INPUTS = $(foreach bench,$(CROSS_BENCHES), $(foreach cat, $(CROSS_CATS), $(bench)$(cat)))

	.SECONDARY: $(LEAN_INPUTS:%=%.out) $(LEAN_INPUTS:%=bench/%.bench) $(CROSS_INPUTS:%=%.out) $(CROSS_INPUTS:%=bench/%.bench)
	.DELETE_ON_ERROR:

	all: report_lean.tex report_cross.tex report

	# disable some built-in rules
	%.lean:
	%.out: %

	%.lean.c: %.lean
	$(LEAN_BIN)/lean --c=$@ $(LEAN_FLAGS) $<
	%.lean.out: %.lean.c
	$(LEAN_BIN)/leanc $(LEANC_FLAGS) -o $@ $<
	# Binaries x.lean.out and x.gcc.lean.out etc. are produced by the
	# same rules and x.lean source file by copying the latter to
	# x.gcc.lean. This also avoids conflicts between intermediate
	# files of the two binaries.
	%.gcc.lean.out: LEAN_BIN = $(LEAN_GCC_BIN)
	%.gcc.lean: %.lean; ln -f $< $@
	%.no_reuse.lean.c: LEAN_BIN = $(LEAN_NO_REUSE_BIN)
	%.no_reuse.lean.out: LEAN_BIN = $(LEAN_NO_REUSE_BIN)
	%.no_reuse.lean: %.lean; ln -f $< $@
	%.no_borrow.lean.c: LEAN_BIN = $(LEAN_NO_BORROW_BIN)
	%.no_borrow.lean.out: LEAN_BIN = $(LEAN_NO_BORROW_BIN)
	%.no_borrow.lean: %.lean; ln -f $< $@
	%.no_st.lean.out: LEAN_BIN = $(LEAN_NO_ST_BIN)
	%.no_st.lean: %.lean; ln -f $< $@

	%.hs.out: %.hs
	$(GHC) $(GHC_FLAGS) -rtsopts $< -o $@
	%.llvm.hs.out: GHC_FLAGS += -fllvm
	%.llvm.hs: %.hs; ln -f $< $@
	%.strict.hs.out: GHC_FLAGS += -XStrict
	%.strict.hs: %.hs; ln -f $< $@

	binarytrees.hs: binarytrees.ghc-6.hs; ln -f $< $@
	# NOTE: changed `-N4` rtsopt to `-N` to be less system-dependent
	binarytrees%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -threaded -rtsopts -with-rtsopts='-N -K128M -H'
	binarytrees.st%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -rtsopts -with-rtsopts='-K128M -H'

	%.ml.out: %.ml
	$(OCAML) $(OCAML_FLAGS) $< -o $@
	%.flambda.ml.out: OCAML = $(OCAML_FLAMBDA)
	%.flambda.ml: %.ml; ln -f $< $@

	binarytrees.ml: binarytrees5_multicore.ml; ln -f $< $@
	binarytrees.st.ml: binarytrees5.ml; ln -f $< $@
	binarytrees%ml.out: OCAML_FLAGS += -noassert -fPIC -nodynlink -inline 100 -O3 -package domainslib -linkpkg
	binarytrees%ml.out: OCAML = ocamlfind ocamlopt

	binarytrees.st.sml: binarytrees.st.mlton-2.sml; ln -f $< $@
	%.mlton.out: %.sml
	$(MLTON_BIN)/mlton $(MLTON_FLAGS) -output $@ $<
	%.gc.mlton.out: MLTON_FLAGS = -profile time
	%.gc.sml: %.sml; ln -f $< $@

	%.mlkit.out: %.sml
	$(MLKIT) $(MLKIT_FLAGS) -o $@ $<

	%.swift.out: %.swift
	$(SWIFTC) $(SWIFTC_FLAGS) -o $@ $<

	binarytrees%swift.out: SWIFTC_FLAGS = -Ounchecked -I Include/swift/apr

	bench:
	-@mkdir bench

	bench/%.bench: %.out \| bench
	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS) -d $< "./$< $(BENCH_PARAMS)" --out $@

	bench/parser.%.bench: BENCH_PARAMS = $(PARSER_TEST_FILE) 50

	bench/binarytrees.%.bench: BENCH_PARAMS = 21
	bench/binarytrees.ml.bench: BENCH_PARAMS = $$(nproc) 21
	bench/binarytrees.gc.ml.bench: BENCH_PARAMS = $$(nproc) 21

	bench/deriv.%.bench: BENCH_PARAMS = 10

	bench/const_fold.%.bench: BENCH_PARAMS = 23

	bench/qsort.%.bench: BENCH_PARAMS = 400

	bench/rbmap.%.bench: BENCH_PARAMS = 2000000

	rbmap_1.%.out: rbmap_checkpoint.%.out; ln -f $< $@
	bench/rbmap_1.%.bench: BENCH_PARAMS = 2000000 1

	rbmap_2.%.out: rbmap_checkpoint.%.out; ln -f $< $@
	bench/rbmap_2.%.bench: BENCH_PARAMS = 2000000 2

	rbmap_10.%.out: rbmap_checkpoint.%.out; ln -f $< $@
	bench/rbmap_10.%.bench: BENCH_PARAMS = 2000000 10

	bench/unionfind.%.bench: BENCH_PARAMS = 3000000

	bench/%gc.lean.bench: %lean.out \| bench
	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
	-d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio -S 'lean_inc_ref_cold,lean_inc_ref_n_cold,lean_dec_ref_cold,lean_del_core,lean_free_small' \| ./lean-gc.py"\
	--runner output --out $@

	bench/%gc.hs.bench: %hs.out \| bench
	$(TEMCI) short exec $(TEMCI_FLAGS)\
	-d $< "./$< +RTS -t --machine-readable -RTS $(BENCH_PARAMS) 2>&1 >/dev/null \| ./ghc-gc.py"\
	--runner output --out $@

	bench/%gc.ml.bench: %ml.out \| bench
	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
	-d $< "time -ao $@.tmp -f '%U' olly trace $@.tmp './$< $(BENCH_PARAMS)' > /dev/null && ./ocaml-gc.py < $@.tmp"\
	--runner output --out $@

	bench/%gc.mlton.bench: %gc.mlton.out \| bench
	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
	-d $< './$< $(BENCH_PARAMS) >/dev/null && $(MLTON_BIN)/mlprof ./$< mlmon.out \| awk '\''$$1 == "<gc>" { print "gc: " substr($$2, 0, length($$2)-1); found = 1 } END { if (found != 1) { print "gc: 0.000001" } }'\'\
	--runner output --out $@

	bench/%gc.mlkit.bench: %gc.mlkit.out \| bench
	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
	-d $< "time -f '%e' ./$< -report_gc $(BENCH_PARAMS) 2>&1 >/dev/null \| ./mlkit-gc.py"\
	--runner output --out $@

	bench/%gc.swift.bench: %swift.out \| bench
	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
	-d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio \| grep -E 'retain\|release\|dealloc\|free' \| ./lean-gc.py"\
	--runner output --out $@

	bench/%.perf.bench: %.out \| bench
	ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
	-d $< "echo > $@.tmp && time -af '%e' perf stat -e cache-misses -x ';' ./$< $(BENCH_PARAMS) 2>&1 >/dev/null \| ./perf.py"\
	--runner output --out $@

	# no benchmarksgame versions
	bench/binarytrees.mlton.bench: ; touch $@
	bench/binarytrees.gc.mlton.bench: ; touch $@
	bench/binarytrees.mlton.perf.bench: ; touch $@
	bench/binarytrees.mlkit.bench: ; touch $@
	bench/binarytrees.gc.mlkit.bench: ; touch $@
	bench/binarytrees.mlkit.perf.bench: ; touch $@

	bench_lean: $(LEAN_INPUTS:%=bench/%.bench)
	bench_cross: $(CROSS_INPUTS:%=bench/%.bench)

	report/lean: ; -@mkdir -p $@
	report/cross: ; -@mkdir -p $@

	report/lean/%: $(foreach cat, $(LEAN_CATS), bench/%$(cat).bench) \| report/lean
	cat $^ > $@.tmp
	$(TEMCI) report $@.tmp --settings cross.yaml --reporter html2 --html2_out $@ --html2_force_override --properties etime

	report/cross/%: $(foreach cat, $(CROSS_TIME_CATS), bench/%$(cat).bench) \| report/cross
	cat $^ > $@.tmp
	$(TEMCI) report $@.tmp --settings cross.yaml --reporter html2 --html2_out $@ --html2_force_override --properties etime

	report/index.html: $(LEAN_BENCHES:%=report/lean/%) $(CROSS_BENCHES:%=report/cross/%)
	echo "<html><h1>Lean variant benchmarks</h1><ul>"> $@
	for bench in $(LEAN_BENCHES); do \
	echo "<li><a href='lean/$$bench/report.html'>$$bench</a><br><img src='lean/$$bench/fig___SinglesProperty______etime___25.0.svg'></img></li>" >> $@; \
	done
	echo "</ul><h1>Cross-language benchmarks</h1><ul>">> $@
	for bench in $(CROSS_BENCHES); do \
	echo "<li><a href='cross/$$bench/report.html'>$$bench</a><br><img src='cross/$$bench/fig___SinglesProperty______etime___25.0.svg'></img></li>" >> $@; \
	done
	echo "</ul></html>">> $@

	report: report/index.html

	# yes.
	space = $() $()

	report_lean.csv report_lean_rbmap.csv: bench_lean
	BENCHES=$(subst $(space),:,$(LEAN_BENCHES)) CATS=$(subst $(space),:,$(LEAN_CATS)) ./report.py report_lean
	column -s';' -t < $@

	report_cross.csv report_cross_rbmap.csv: bench_cross
	BENCHES=$(subst $(space),:,$(CROSS_BENCHES)) CATS=$(subst $(space),:,$(CROSS_CATS)) ./report.py report_cross
	column -s';' -t < $@

	TO_TEX = 's/-/---/g;s/%/\\%/g;s/\.$$//g;s/(\([0-9]\)\([0-9]\))/\\ensuremath{\\tilde{\1}\\tilde{\2}}/g;s/(\([0-9]\))/\\ensuremath{\\tilde{\1}}/g;s/;/ \& /g;s/$$/\\\\/'
	report_%.tex: report_%.csv
	tail -n +2 $< \| head -n -1 \| sed 's/^\([^;]\+\)/\\verb!\1!/;'$(TO_TEX) > $@
	echo -n '\midrule ' >> $@
	tail -n 1 $< \| sed 's/. /.\\ /;'$(TO_TEX) >> $@

	clean:
	-rm .out bench/